feat: release changes for version 0.4.4

Author: DeXtroTip

galaxy/core/galaxy.py

@@ -2,6 +2,7 @@
 import importlib
 import json
 import logging
+import time
 import traceback
 from collections import OrderedDict
 from collections.abc import AsyncGenerator, Callable
@@ -146,7 +147,13 @@ async def run_integration_methods(
     async def _run_integration_method_and_push_to_magneto(method: Callable) -> None:
         logger.info("%-*s | Executing method", method_logger_width, method.__name__)
 
+        start_time = time.time()
         method_entities = [e async for e in _run_integration_method_to_entities(instance=instance, method=method)]
+        end_time = time.time()
+        logger.debug(
+            "%-*s | Execution: %d ms", method_logger_width, method.__name__, int((end_time - start_time) * 1000)
+        )
+
         logger.info("%-*s | Entities found: %d", method_logger_width, method.__name__, len(method_entities))
         logger.debug("%-*s | Results: %r", method_logger_width, method.__name__, method_entities)
 

galaxy/core/models.py

@@ -1,7 +1,7 @@
-from typing import Any, Literal
+from typing import Any, List, Literal
 
 import yaml
-from pydantic import BaseModel, ConfigDict, Field
+from pydantic import BaseModel, ConfigDict, Field, RootModel
 
 __all__ = ["Config", "ExecutionType", "RelyConfig", "IntegrationConfig", "SchedulerJobStates"]
 
@@ -51,3 +51,19 @@ def from_yaml(cls, file_path):
 
     rely: RelyConfig
     integration: IntegrationConfig
+
+
+class FileCheck(BaseModel):
+    path: str
+    destination: str
+    regex: str = Field(..., alias="regex")
+
+
+class FileCheckList(RootModel):
+    root: List[FileCheck]
+
+    def __iter__(self):
+        return iter(self.root)
+
+    def __getitem__(self, item):
+        return self.root[item]

galaxy/core/utils.py

@@ -9,9 +9,11 @@
 from aiohttp import ClientResponseError
 from fastapi import HTTPException, Request, Security
 from fastapi.security.api_key import APIKeyHeader
+from pydantic import ValidationError
 
 from galaxy.core.magneto import Magneto
 from galaxy.core.mapper import Mapper
+from galaxy.core.models import Config, FileCheckList
 
 
 def from_env(variable, raise_exception=False) -> str | int:
@@ -94,3 +96,37 @@ async def get_api_key(api_key_header: str = Security(APIKeyHeader(name="authoriz
         return True
     else:
         raise HTTPException(status_code=403, detail="Could not validate credentials")
+
+
+def files_to_check(config: Config, logger: logging.Logger) -> list[dict]:
+    files_to_check: list | str | FileCheckList = config.integration.properties.get("filesToCheck", "")
+
+    if isinstance(files_to_check, str):
+        try:
+            files_check_list = FileCheckList.model_validate_json(files_to_check)
+        except ValidationError as e:
+            files_from_str = [
+                {"path": fc[0], "destination": fc[1], "regex": fc[2]}
+                for fc in (file_check.split("::") for file_check in files_to_check.split("||"))
+                if len(fc) == 3 and fc[0] and fc[1]
+            ]
+
+            try:
+                files_check_list = FileCheckList.model_validate(files_from_str)
+            except ValidationError:
+                logger.error(f"Invalid file check config: {files_to_check}: {e}")
+                return []
+
+        return list([fc.model_dump() for fc in files_check_list])
+
+    if isinstance(files_to_check, list):
+        try:
+            files_check_list = FileCheckList.model_validate(files_to_check)
+
+            return list([fc.model_dump() for fc in files_check_list])
+        except ValidationError:
+            logger.error(f"Invalid file check config: {files_to_check}")
+            return []
+
+    logger.warning(f"Invalid file check config: {files_to_check}")
+    return []

galaxy/integrations/github/.rely/automations.json

@@ -251,7 +251,9 @@
               "closedAt": "{{ data.properties.closedAt if data.properties.closedAt else data.properties.mergedAt }}",
               "createdBy": "{{ data.relations.creator.value }}",
               "reviewers": "{{ data.relations.reviewers.value if data.relations.reviewers else [] }}",
-              "firstReviewActivityAt": "{{ [data.properties.firstReviewAt, data.properties.closedAt] | select() | min | default(none) }}"
+              "firstReviewActivityAt": "{{ [data.properties.firstReviewAt, data.properties.closedAt] | select() | min | default(none) }}",
+              "additions": "{{ data.properties.additions if data.properties.additions else none }}",
+              "deletions": " {{ data.properties.deletions if data.properties.deletions else none }}"
             }
           },
           "resourceType": "entity",

galaxy/integrations/github/.rely/blueprints.json

@@ -222,6 +222,16 @@
           "format": "date-time",
           "title": "First Review At",
           "description": "The date of the first review activity on the pull request."
+        },
+        "additions": {
+          "type": "integer",
+          "title": "Number of Additions",
+          "description": "The number of lines added in the pull request."
+        },
+        "deletions": {
+          "type": "integer",
+          "title": "Number of Deletions",
+          "description": "The number of lines deleted in the pull request."
         }
       }
     },
@@ -730,5 +740,52 @@
         "array": false
       }
     }
+  },
+  {
+    "id": "github.v1.repository_metrics",
+    "title": "GitHub Repository Metrics",
+    "description": "Blueprint defining GitHub Repository Metrics",
+    "icon": "github",
+    "schemaProperties": {
+      "title": "Blueprint properties",
+      "type": "object",
+      "properties": {
+        "metrics": {
+          "type": "object",
+          "title": "Metrics",
+          "description": "Repository metrics data",
+          "properties": {
+            "dailyCommits": {
+              "type": "array",
+              "title": "Daily Commit Metrics",
+              "description": "Daily commit counts for the repository",
+              "items": {
+                "type": "object",
+                "properties": {
+                  "date": {
+                    "type": "string",
+                    "title": "Date",
+                    "description": "The date of the metric"
+                  },
+                  "count": {
+                    "type": "number",
+                    "title": "Count",
+                    "description": "Number of commits for this date"
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    },
+    "relations": {
+      "repository": {
+        "value": "github.v1.repository",
+        "title": "Repository",
+        "description": "The project this metric belongs to",
+        "array": false
+      }
+    }
   }
 ]

galaxy/integrations/github/.rely/mappings.yaml

@@ -78,6 +78,8 @@ resources:
         | sort
         | first
         // null
+    additions: .additions
+    deletions: .deletions
     relations:
       repository:
         value: .context.repositoryId | tostring
@@ -221,3 +223,20 @@ resources:
         value: .context.environmentId | tostring
       triggeredBy:
         value: .creator.login
+
+- kind: repository_metrics
+  mappings:
+    id: '.context.repositoryId | tostring + "-metrics"'
+    title: '.context.repositoryName + " (Metrics)"'
+    blueprintId: '"github.v1.repository_metrics"'
+    properties:
+      metrics:
+        dailyCommits: |
+          with_entries(select(.key == "commits"))
+          | .commits
+          | map(. + {date: .committedDate | fromdate | strftime("%Y-%m-%d")})
+          | group_by(.date)
+          | map({date: .[0].date, count: length})
+    relations:
+      repository:
+        value: .context.repositoryId | tostring

galaxy/integrations/github/client.py

@@ -1,6 +1,7 @@
 import base64
 from collections.abc import Iterable
 from datetime import datetime, timedelta
+from functools import partial
 from itertools import count
 from types import TracebackType
 from typing import Any
@@ -432,3 +433,42 @@ async def get_deployments(
             cursor = page_info["endCursor"]
 
         return all_deployments
+
+    async def get_commits(
+        self, organization: str, repo: str, branch: str, *, exclude_merge_commits: bool = True
+    ) -> list[dict[str, str | int]]:
+        all_commits = []
+
+        query_builder = partial(
+            build_graphql_query,
+            query_type=QueryType.COMMITS,
+            repo_id=self.build_repo_id(organization, repo),
+            branch=branch,
+            page_size=self.page_size,
+            since=self.history_limit_timestamp,
+        )
+
+        cursor = None
+        while True:
+            query = query_builder(after=cursor)
+            response = await self._make_graphql_request(query)
+
+            try:
+                data = response["data"]["repository"]["commits"]["history"] or {}
+                commits = data["nodes"] or []
+                all_commits.extend(
+                    [commit for commit in commits if not exclude_merge_commits or not self._is_merge_commit(commit)]
+                )
+                page_info = data["pageInfo"] or {}
+                has_next_page = page_info.get("hasNextPage") or False
+                cursor = page_info.get("endCursor")
+            except (KeyError, TypeError):
+                break
+
+            if not has_next_page:
+                break
+
+        return all_commits
+
+    def _is_merge_commit(self, commit: dict) -> bool:
+        return ((commit.get("parents") or {}).get("totalCount") or 0) > 1

galaxy/integrations/github/main.py

@@ -84,13 +84,15 @@ async def repository(self) -> None:
         )
 
         for metadata in repositories_metadata:
+            content = await self.client.get_repo(metadata["owner"]["login"], metadata["name"])
             self.repositories[metadata["id"]] = {
                 "id": metadata["id"],
                 "slug": metadata["name"],
                 "owner": metadata["owner"]["login"],
                 "link": metadata["html_url"],
                 "metadata": metadata,
-                "content": await self.client.get_repo(metadata["owner"]["login"], metadata["name"]),
+                "default_branch": content.get("defaultBranchRef", {}).get("name"),
+                "content": content,
             }
 
         self.logger.info(f"Found {len(self.repositories)} repositories")
@@ -284,6 +286,23 @@ async def deployments(self) -> list[dict]:
 
         return new_entities + deployments_mapped
 
+    @register(_methods, group=6)
+    async def repository_metrics(self) -> list[dict]:
+        all_metrics = []
+        for repo in self.repositories.values():
+            commits = await self.client.get_commits(repo["owner"], repo["slug"], branch=repo["default_branch"])
+            repository_metrics = await self.mapper.process(
+                "repository_metrics",
+                [{"commits": commits}],
+                context={"repositoryId": repo["id"], "repositoryName": repo["slug"]},
+            )
+            all_metrics.extend(repository_metrics)
+
+        self.logger.info(
+            f"Calculated {len(all_metrics)} repository metrics from the last {self.client.days_of_history} days"
+        )
+        return all_metrics
+
     async def register_inactive_users(self, inactive_usernames):
         if not inactive_usernames:
             return []

galaxy/integrations/github/queries.py

@@ -1,4 +1,5 @@
 from collections.abc import Iterable
+from datetime import datetime
 from enum import Enum
 from typing import Any, Optional
 
@@ -15,6 +16,7 @@ class QueryType(str, Enum):
     TEAMS = "teams"
     TEAM_MEMBERS = "team_members"
     TEAM_REPOS = "team_repos"
+    COMMITS = "commits"
 
 
 def build_graphql_query(query_type: QueryType, **params: Any) -> dict:
@@ -27,6 +29,7 @@ def build_graphql_query(query_type: QueryType, **params: Any) -> dict:
         QueryType.TEAMS: _build_teams_query,
         QueryType.TEAM_MEMBERS: _build_team_members,
         QueryType.TEAM_REPOS: _build_team_repos,
+        QueryType.COMMITS: _build_commits_query,
     }
 
     query, variables = builders_by_query_type[query_type](**params)
@@ -176,6 +179,8 @@ def _build_pull_requests_query(
                                     login
                                 }
                             }
+                            additions
+                            deletions
                         }
                     }
                     pageInfo {
@@ -469,3 +474,53 @@ def _build_members_query(
 
     variables = {"owner": owner, "after": after, "pageSize": page_size}
     return query, variables
+
+
+def _build_commits_query(
+    repo_id: str, branch: str, *, after: str | None = None, since: datetime | None = None, page_size: int = 50
+) -> tuple[str, dict[str, str]]:
+    query = """
+        query GetCommits(
+            $owner: String!
+            $name: String!
+            $branch: String!
+            $since: GitTimestamp
+            $after: String = null
+            $pageSize: Int = 50
+        ) {
+            repository(owner: $owner, name: $name, followRenames: true) {
+                commits: object(expression: $branch) {
+                    ... on Commit {
+                        history(first: $pageSize, after: $after, since: $since) {
+                            pageInfo {
+                                endCursor
+                                hasNextPage
+                            }
+                            nodes {
+                                oid
+                                committedDate
+                                author {
+                                    user {
+                                        login
+                                    }
+                                }
+                                parents {
+                                    totalCount
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    """
+    owner, name = repo_id.split("/", maxsplit=1)
+    variables = {
+        "owner": owner,
+        "name": name,
+        "branch": branch,
+        "after": after,
+        "pageSize": page_size,
+        "since": since.isoformat() if since is not None else None,
+    }
+    return query, variables