From 67f05859aafc2fbed711ef4c8e66dee23bcc222d Mon Sep 17 00:00:00 2001 From: Zakaria Mansouri Date: Thu, 5 Sep 2024 21:39:35 +0200 Subject: [PATCH 01/12] added contributions to database --- api/db/migrations/0001_warm_lockheed.sql | 14 ++ api/db/migrations/meta/0001_snapshot.json | 234 ++++++++++++++++++ api/db/migrations/meta/_journal.json | 7 + api/src/contribution/repository.ts | 4 + api/src/digest/cron.ts | 24 +- api/src/github/dto.ts | 41 ++- api/src/github/service.ts | 21 +- .../models/src/contribution/index.spec.ts | 21 +- packages/models/src/contribution/index.ts | 50 ++-- 9 files changed, 366 insertions(+), 50 deletions(-) create mode 100644 api/db/migrations/0001_warm_lockheed.sql create mode 100644 api/db/migrations/meta/0001_snapshot.json diff --git a/api/db/migrations/0001_warm_lockheed.sql b/api/db/migrations/0001_warm_lockheed.sql new file mode 100644 index 000000000..9df3ca887 --- /dev/null +++ b/api/db/migrations/0001_warm_lockheed.sql @@ -0,0 +1,14 @@ +CREATE TABLE `contributions` ( + `id` integer PRIMARY KEY AUTOINCREMENT NOT NULL, + `record_imported_at` text DEFAULT CURRENT_TIMESTAMP NOT NULL, + `name` text NOT NULL, + `updated_at` text NOT NULL, + `url` text NOT NULL, + `type` text NOT NULL, + `run_id` text NOT NULL, + `activity_count` integer NOT NULL, + `repository_id` integer NOT NULL, + FOREIGN KEY (`repository_id`) REFERENCES `repositories`(`id`) ON UPDATE no action ON DELETE no action +); +--> statement-breakpoint +CREATE UNIQUE INDEX `contributions_url_unique` ON `contributions` (`url`); \ No newline at end of file diff --git a/api/db/migrations/meta/0001_snapshot.json b/api/db/migrations/meta/0001_snapshot.json new file mode 100644 index 000000000..7930c5c7e --- /dev/null +++ b/api/db/migrations/meta/0001_snapshot.json @@ -0,0 +1,234 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "d333591a-11be-4b30-89f8-4da0aa5bd906", + "prevId": "ab1b512a-8ca8-44e1-b497-445465117250", + "tables": { + "contributions": { + "name": "contributions", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "record_imported_at": { + "name": "record_imported_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "activity_count": { + "name": "activity_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "repository_id": { + "name": "repository_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "contributions_url_unique": { + "name": "contributions_url_unique", + "columns": ["url"], + "isUnique": true + } + }, + "foreignKeys": { + "contributions_repository_id_repositories_id_fk": { + "name": "contributions_repository_id_repositories_id_fk", + "tableFrom": "contributions", + "tableTo": "repositories", + "columnsFrom": ["repository_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "projects": { + "name": "projects", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "record_imported_at": { + "name": "record_imported_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'initial-run-id'" + } + }, + "indexes": { + "projects_slug_unique": { + "name": "projects_slug_unique", + "columns": ["slug"], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "repositories": { + "name": "repositories", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "record_imported_at": { + "name": "record_imported_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "owner": { + "name": "owner", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'initial-run-id'" + }, + "project_id": { + "name": "project_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "repositories_provider_owner_name_unique": { + "name": "repositories_provider_owner_name_unique", + "columns": ["provider", "owner", "name"], + "isUnique": true + } + }, + "foreignKeys": { + "repositories_project_id_projects_id_fk": { + "name": "repositories_project_id_projects_id_fk", + "tableFrom": "repositories", + "tableTo": "projects", + "columnsFrom": ["project_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + } + }, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} diff --git a/api/db/migrations/meta/_journal.json b/api/db/migrations/meta/_journal.json index a87ceb45e..68fd83398 100644 --- a/api/db/migrations/meta/_journal.json +++ b/api/db/migrations/meta/_journal.json @@ -8,6 +8,13 @@ "when": 1725462108723, "tag": "0000_new_lake", "breakpoints": true + }, + { + "idx": 1, + "version": "6", + "when": 1725564569182, + "tag": "0001_warm_lockheed", + "breakpoints": true } ] } diff --git a/api/src/contribution/repository.ts b/api/src/contribution/repository.ts index bb0dff878..6f19d9da3 100644 --- a/api/src/contribution/repository.ts +++ b/api/src/contribution/repository.ts @@ -1,3 +1,7 @@ +// @TODO-ZM: remove this +// eslint-disable-next-line @typescript-eslint/ban-ts-comment +// @ts-nocheck + import { Model } from "@dzcode.io/models/dist/_base"; import { ContributionEntity } from "@dzcode.io/models/dist/contribution"; import { DataService } from "src/data/service"; diff --git a/api/src/digest/cron.ts b/api/src/digest/cron.ts index 344122540..646dfea98 100644 --- a/api/src/digest/cron.ts +++ b/api/src/digest/cron.ts @@ -1,5 +1,6 @@ import { captureException, cron } from "@sentry/node"; import { CronJob } from "cron"; +import { ContributionRepository } from "src/contribution/repository"; import { DataService } from "src/data/service"; import { GithubService } from "src/github/service"; import { LoggerService } from "src/logger/service"; @@ -18,6 +19,7 @@ export class DigestCron { private readonly githubService: GithubService, private readonly projectsRepository: ProjectRepository, private readonly repositoriesRepository: RepositoryRepository, + private readonly contributionsRepository: ContributionRepository, ) { const SentryCronJob = cron.instrumentCron(CronJob, "DigestCron"); new SentryCronJob( @@ -74,7 +76,6 @@ export class DigestCron { repo: repository.name, }); - // eslint-disable-next-line @typescript-eslint/no-unused-vars const [{ id: repositoryId }] = await this.repositoriesRepository.upsert({ provider: "github", name: repoInfo.name, @@ -82,6 +83,26 @@ export class DigestCron { runId, projectId, }); + + const issues = await this.githubService.listRepositoryIssues({ + owner: repository.owner, + repo: repository.name, + }); + + for (const issue of issues.issues) { + const type = issue.pull_request ? "PULL_REQUEST" : "ISSUE"; + const [{ id: contributionId }] = await this.contributionsRepository.upsert({ + title: issue.title, + type, + updatedAt: issue.updated_at, + activityCount: issue.comments, + runId, + repositoryId, + url: type === "PULL_REQUEST" ? issue.pull_request.html_url : issue.html_url, + }); + + console.log("contributionId", contributionId); + } } catch (error) { // @TODO-ZM: capture error console.error(error); @@ -93,6 +114,7 @@ export class DigestCron { } } + await this.contributionsRepository.deleteAllButWithRunId(runId); await this.repositoriesRepository.deleteAllButWithRunId(runId); await this.projectsRepository.deleteAllButWithRunId(runId); diff --git a/api/src/github/dto.ts b/api/src/github/dto.ts index 3c5b64377..e190897d4 100644 --- a/api/src/github/dto.ts +++ b/api/src/github/dto.ts @@ -1,4 +1,4 @@ -import { IsString, Validate, ValidateNested } from "class-validator"; +import { IsIn, IsNumber, IsString, Validate, ValidateNested } from "class-validator"; import { IsMapOfStringNumber } from "src/_utils/validator/is-map-of-string-number"; export class GitHubListRepositoryLanguagesResponse { @@ -18,3 +18,42 @@ export class GetRepositoryResponse { @ValidateNested() owner!: GithubAccount; } + +class GetRepositoryIssuesPullRequestResponse { + @IsString() + html_url!: string; // eslint-disable-line camelcase +} + +class GetRepositoryIssuesResponse { + @IsString() + title!: string; + + @ValidateNested() + user!: GithubAccount; + + @IsString({ each: true }) + labels!: string[]; + + @IsIn(["closed", "open"]) + state!: "closed" | "open"; + + @ValidateNested({ each: true }) + assignees!: GithubAccount[]; + + @IsString() + updated_at!: string; // eslint-disable-line camelcase + + @IsString() + html_url!: string; // eslint-disable-line camelcase + + @ValidateNested() + pull_request!: GetRepositoryIssuesPullRequestResponse; // eslint-disable-line camelcase + + @IsNumber() + comments!: number; +} + +export class GetRepositoryIssuesResponseArray { + @ValidateNested({ each: true }) + issues!: GetRepositoryIssuesResponse[]; +} diff --git a/api/src/github/service.ts b/api/src/github/service.ts index d0455fdbe..45b8fb8cb 100644 --- a/api/src/github/service.ts +++ b/api/src/github/service.ts @@ -4,13 +4,15 @@ import { ConfigService } from "src/config/service"; import { FetchService } from "src/fetch/service"; import { Service } from "typedi"; -import { GetRepositoryResponse, GitHubListRepositoryLanguagesResponse } from "./dto"; +import { + GetRepositoryIssuesResponseArray, + GetRepositoryResponse, + GitHubListRepositoryLanguagesResponse, +} from "./dto"; import { GeneralGithubQuery, GetRepositoryInput, GetUserInput, - GithubIssue, - GitHubListRepositoryIssuesInput, GitHubListRepositoryLanguagesInput, GitHubListRepositoryMilestonesInput, GithubMilestone, @@ -58,17 +60,20 @@ export class GithubService { public listRepositoryIssues = async ({ owner, - repository, - }: GitHubListRepositoryIssuesInput): Promise => { - const issues = await this.fetchService.getUnsafe( - `${this.apiURL}/repos/${owner}/${repository}/issues`, + repo, + }: GetRepositoryInput): Promise => { + const repoIssues = await this.fetchService.get( + `${this.apiURL}/repos/${owner}/${repo}/issues`, { headers: this.githubToken ? { Authorization: `Token ${this.githubToken}` } : {}, + // @TODO-ZM: add pagination params: { sort: "updated", per_page: 100 }, // eslint-disable-line camelcase }, + GetRepositoryIssuesResponseArray, + "issues", ); - return issues; + return repoIssues; }; public listRepositoryLanguages = async ({ diff --git a/packages/models/src/contribution/index.spec.ts b/packages/models/src/contribution/index.spec.ts index 1a25692fb..3e90685be 100644 --- a/packages/models/src/contribution/index.spec.ts +++ b/packages/models/src/contribution/index.spec.ts @@ -5,26 +5,13 @@ import { ContributionEntity } from "."; runDTOTestCases( ContributionEntity, { - commentsCount: 0, - id: "71", - labels: ["discussion", "good first issue"], - languages: ["JavaScript", "Shell"], - project: { - name: "Leblad", - slug: "Leblad", - }, + activityCount: 0, title: "Update the data set", - type: "issue", - createdAt: "2020-02-02T20:22:02.000Z", + type: "ISSUE", updatedAt: "2020-02-02T20:22:02.000Z", url: "https://github.com/dzcode-io/leblad/issues/71", - createdBy: { - id: "github/20110076", - username: "ZibanPirate", - name: "Zakaria Mansouri", - profileUrl: "/Account/github/20110076", - avatarUrl: "https://avatars.githubusercontent.com/u/20110076?v=4", - }, + id: 0, + runId: "test-run-id", }, {}, ); diff --git a/packages/models/src/contribution/index.ts b/packages/models/src/contribution/index.ts index da22a1118..78602e086 100644 --- a/packages/models/src/contribution/index.ts +++ b/packages/models/src/contribution/index.ts @@ -1,42 +1,46 @@ import { Type } from "class-transformer"; -import { IsDateString, IsNumber, IsString, IsUrl, ValidateNested } from "class-validator"; +import { IsDateString, IsIn, IsNumber, IsString, IsUrl, ValidateNested } from "class-validator"; import { BaseEntity, Model } from "src/_base"; import { AccountEntity } from "src/account"; import { ProjectReferenceEntity } from "src/project-reference"; -export class ContributionEntity extends BaseEntity { +export class ContributionEntityCompact extends BaseEntity { + // @TODO-ZM: move this to BaseEntity @IsString() - id!: string; + id!: number; @IsString() title!: string; - @ValidateNested() - @Type(() => ProjectReferenceEntity) - project!: Model; - - @ValidateNested() - @Type(() => AccountEntity) - createdBy!: Model; - - @IsString() - type!: "issue" | "pullRequest"; + @IsIn(["ISSUE", "PULL_REQUEST"]) + type!: "ISSUE" | "PULL_REQUEST"; @IsUrl() url!: string; - @IsString({ each: true }) - languages!: string[]; - - @IsString({ each: true }) - labels!: string[]; - - @IsDateString() - createdAt!: string; - @IsDateString() updatedAt!: string; @IsNumber() - commentsCount!: number; + activityCount!: number; +} +export class ContributionEntity extends ContributionEntityCompact { + @IsString() + runId!: string; +} + +export class ContributionEntityForList extends BaseEntity { + @ValidateNested() + @Type(() => AccountEntity) + createdBy!: Model; // Compact + + @ValidateNested() + @Type(() => ProjectReferenceEntity) + project!: Model; // Compact + + @IsString({ each: true }) + languages!: string[]; // Compact + + @IsString({ each: true }) + labels!: string[]; // Compact } From 9f57d53483d02e2d39278d7238c918df48e933d0 Mon Sep 17 00:00:00 2001 From: Zakaria Mansouri Date: Fri, 6 Sep 2024 21:24:18 +0200 Subject: [PATCH 02/12] CRUD contributions --- api/db/migrations/0002_mighty_may_parker.sql | 2 + api/db/migrations/meta/0002_snapshot.json | 234 ++++++++++++++++++ api/db/migrations/meta/_journal.json | 7 + api/src/_utils/case.ts | 11 + api/src/_utils/reverse-hierarchy.ts | 46 ++++ api/src/_utils/unstringify-deep.ts | 34 +++ api/src/app/types/index.ts | 2 - api/src/contribution/repository.ts | 174 +++++-------- api/src/contribution/table.ts | 25 ++ .../__snapshots__/index.spec.ts.snap | 155 ------------ .../models/src/contribution/index.spec.ts | 17 -- packages/models/src/contribution/index.ts | 53 +--- packages/models/src/project/index.ts | 30 +-- .../__snapshots__/index.spec.ts.snap | 64 ----- packages/models/src/repository/index.spec.ts | 18 -- packages/models/src/repository/index.ts | 44 +--- 16 files changed, 438 insertions(+), 478 deletions(-) create mode 100644 api/db/migrations/0002_mighty_may_parker.sql create mode 100644 api/db/migrations/meta/0002_snapshot.json create mode 100644 api/src/_utils/case.ts create mode 100644 api/src/_utils/reverse-hierarchy.ts create mode 100644 api/src/_utils/unstringify-deep.ts create mode 100644 api/src/contribution/table.ts delete mode 100644 packages/models/src/contribution/__snapshots__/index.spec.ts.snap delete mode 100644 packages/models/src/contribution/index.spec.ts delete mode 100644 packages/models/src/repository/__snapshots__/index.spec.ts.snap delete mode 100644 packages/models/src/repository/index.spec.ts diff --git a/api/db/migrations/0002_mighty_may_parker.sql b/api/db/migrations/0002_mighty_may_parker.sql new file mode 100644 index 000000000..7095e1b0f --- /dev/null +++ b/api/db/migrations/0002_mighty_may_parker.sql @@ -0,0 +1,2 @@ +ALTER TABLE `contributions` ADD `title` text NOT NULL;--> statement-breakpoint +ALTER TABLE `contributions` DROP COLUMN `name`; \ No newline at end of file diff --git a/api/db/migrations/meta/0002_snapshot.json b/api/db/migrations/meta/0002_snapshot.json new file mode 100644 index 000000000..29b79923b --- /dev/null +++ b/api/db/migrations/meta/0002_snapshot.json @@ -0,0 +1,234 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "4c5b2a9b-5f9e-496c-a73e-e260a0513b86", + "prevId": "d333591a-11be-4b30-89f8-4da0aa5bd906", + "tables": { + "contributions": { + "name": "contributions", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "record_imported_at": { + "name": "record_imported_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "activity_count": { + "name": "activity_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "repository_id": { + "name": "repository_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "contributions_url_unique": { + "name": "contributions_url_unique", + "columns": ["url"], + "isUnique": true + } + }, + "foreignKeys": { + "contributions_repository_id_repositories_id_fk": { + "name": "contributions_repository_id_repositories_id_fk", + "tableFrom": "contributions", + "tableTo": "repositories", + "columnsFrom": ["repository_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "projects": { + "name": "projects", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "record_imported_at": { + "name": "record_imported_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'initial-run-id'" + } + }, + "indexes": { + "projects_slug_unique": { + "name": "projects_slug_unique", + "columns": ["slug"], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "repositories": { + "name": "repositories", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "record_imported_at": { + "name": "record_imported_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "owner": { + "name": "owner", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'initial-run-id'" + }, + "project_id": { + "name": "project_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "repositories_provider_owner_name_unique": { + "name": "repositories_provider_owner_name_unique", + "columns": ["provider", "owner", "name"], + "isUnique": true + } + }, + "foreignKeys": { + "repositories_project_id_projects_id_fk": { + "name": "repositories_project_id_projects_id_fk", + "tableFrom": "repositories", + "tableTo": "projects", + "columnsFrom": ["project_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + } + }, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} diff --git a/api/db/migrations/meta/_journal.json b/api/db/migrations/meta/_journal.json index 68fd83398..a3a5cb484 100644 --- a/api/db/migrations/meta/_journal.json +++ b/api/db/migrations/meta/_journal.json @@ -15,6 +15,13 @@ "when": 1725564569182, "tag": "0001_warm_lockheed", "breakpoints": true + }, + { + "idx": 2, + "version": "6", + "when": 1725568393657, + "tag": "0002_mighty_may_parker", + "breakpoints": true } ] } diff --git a/api/src/_utils/case.ts b/api/src/_utils/case.ts new file mode 100644 index 000000000..9870d38ad --- /dev/null +++ b/api/src/_utils/case.ts @@ -0,0 +1,11 @@ +import camelCase from "lodash/camelCase"; +import mapKeys from "lodash/mapKeys"; + +export function camelCaseObject>(obj: T): T { + const array = !Array.isArray(obj) ? [obj] : obj; + const camelCasedArray = array.map((item) => { + return mapKeys(item, (value, key) => camelCase(key)); + }); + + return (!Array.isArray(obj) ? camelCasedArray[0] : camelCasedArray) as T; +} diff --git a/api/src/_utils/reverse-hierarchy.ts b/api/src/_utils/reverse-hierarchy.ts new file mode 100644 index 000000000..e1d08e0e8 --- /dev/null +++ b/api/src/_utils/reverse-hierarchy.ts @@ -0,0 +1,46 @@ +type ForeignKeyParentKeyRecord = { from: string; setParentAs: string }; + +export function reverseHierarchy( + _obj: unknown, + foreignKeyParentKeyRecord: ForeignKeyParentKeyRecord[], + parentWithItsKey: Record = {}, +): any { + if (Array.isArray(_obj)) { + return _obj + .map((item) => reverseHierarchy(item, foreignKeyParentKeyRecord, parentWithItsKey)) + .reduce((pV, cV) => [...pV, ...cV], []); + } + + if (typeof _obj !== "object") { + return _obj; + } + + const obj = { ..._obj, ...parentWithItsKey }; + + const objWithRecognizedKeys: Record = {}; + const objWithoutRecognizedKeys: Record = {}; + for (const [key, value] of Object.entries(obj)) { + const mappedKey = foreignKeyParentKeyRecord.find(({ from: mk }) => mk === key)?.from; + if (mappedKey) objWithRecognizedKeys[mappedKey] = value; + else objWithoutRecognizedKeys[key] = value; + } + + if (Object.keys(objWithRecognizedKeys).length > 0) { + let res: unknown[] = []; + for (const recognizedKey in objWithRecognizedKeys) { + if (Object.prototype.hasOwnProperty.call(objWithRecognizedKeys, recognizedKey)) { + const recognizedObj = objWithRecognizedKeys[recognizedKey]; + const { setParentAs } = foreignKeyParentKeyRecord.find( + ({ from }) => from === recognizedKey, + ) as ForeignKeyParentKeyRecord; + const reversedPredecessor = reverseHierarchy(recognizedObj, foreignKeyParentKeyRecord, { + [setParentAs]: objWithoutRecognizedKeys, + }); + res = res.concat(reversedPredecessor); + } + } + return res; + } + + return [objWithoutRecognizedKeys]; +} diff --git a/api/src/_utils/unstringify-deep.ts b/api/src/_utils/unstringify-deep.ts new file mode 100644 index 000000000..61f5bbe36 --- /dev/null +++ b/api/src/_utils/unstringify-deep.ts @@ -0,0 +1,34 @@ +/** + * Recursively look for any string field that starts with `[{"` or `{"` and parse it + unStringify + * its children. + */ + +// eslint-disable-next-line @typescript-eslint/no-unnecessary-type-constraint +export function unStringifyDeep(obj: T): T { + if (Array.isArray(obj)) { + return obj.map((item) => unStringifyDeep(item)) as T; + } + + if (typeof obj !== "object" || obj === null) { + return obj; + } + + const result = { ...obj }; + + for (const key in result) { + if (typeof result[key] === "string") { + try { + const value = JSON.parse(result[key]); + if (typeof value === "object") { + result[key] = unStringifyDeep(value); + } else { + result[key] = value; + } + } catch (error) { + // ignore + } + } + } + + return result as T; +} diff --git a/api/src/app/types/index.ts b/api/src/app/types/index.ts index 868ed103e..2ad143283 100644 --- a/api/src/app/types/index.ts +++ b/api/src/app/types/index.ts @@ -1,5 +1,3 @@ -import "reflect-metadata"; - import { IsNumber, IsObject, IsOptional, IsString } from "class-validator"; export class GeneralResponseDto { diff --git a/api/src/contribution/repository.ts b/api/src/contribution/repository.ts index 6f19d9da3..a98e450bf 100644 --- a/api/src/contribution/repository.ts +++ b/api/src/contribution/repository.ts @@ -1,128 +1,74 @@ -// @TODO-ZM: remove this -// eslint-disable-next-line @typescript-eslint/ban-ts-comment -// @ts-nocheck - -import { Model } from "@dzcode.io/models/dist/_base"; -import { ContributionEntity } from "@dzcode.io/models/dist/contribution"; -import { DataService } from "src/data/service"; -import { GithubService } from "src/github/service"; -import { LoggerService } from "src/logger/service"; +import { ne, sql } from "drizzle-orm"; +import { camelCaseObject } from "src/_utils/case"; +import { reverseHierarchy } from "src/_utils/reverse-hierarchy"; +import { unStringifyDeep } from "src/_utils/unstringify-deep"; +import { projectsTable } from "src/project/table"; +import { repositoriesTable } from "src/repository/table"; +import { SQLiteService } from "src/sqlite/service"; import { Service } from "typedi"; -import { allFilterNames, FilterDto, GetContributionsResponseDto, OptionDto } from "./types"; +import { ContributionRow, contributionsTable } from "./table"; @Service() export class ContributionRepository { - constructor( - private readonly githubService: GithubService, - private readonly dataService: DataService, - private readonly loggerService: LoggerService, - ) {} - - public async find( - filterFn?: (value: ContributionEntity, index: number, array: ContributionEntity[]) => boolean, - ): Promise> { - const projects = await this.dataService.listProjects(); + constructor(private readonly sqliteService: SQLiteService) { + this.findForList(); + } - let contributions = ( - await Promise.all( - projects.reduce[]>[]>( - (pV, { repositories, name, slug }) => [ - ...pV, - ...repositories - .filter(({ provider }) => provider === "github") - .map(async ({ owner, name: repository }) => { - try { - const issuesIncludingPRs = await this.githubService.listRepositoryIssues({ - owner, - repository, - }); + public async upsert(contribution: ContributionRow) { + return await this.sqliteService.db + .insert(contributionsTable) + .values(contribution) + .onConflictDoUpdate({ + target: [contributionsTable.url], + set: contribution, + }) + .returning({ id: contributionsTable.id }); + } - const languages = await this.githubService.listRepositoryLanguages({ - owner, - repository, - }); - return issuesIncludingPRs.map>( - ({ - number, - labels: gLabels, - title, - html_url, // eslint-disable-line camelcase - pull_request, // eslint-disable-line camelcase - created_at, // eslint-disable-line camelcase - updated_at, // eslint-disable-line camelcase - comments, - user, - }) => ({ - id: `${number}`, - labels: gLabels.map(({ name }) => name), - languages: Object.keys(languages), - project: { - slug, - name, - }, - title, - type: pull_request ? "pullRequest" : "issue", // eslint-disable-line camelcase - url: html_url, // eslint-disable-line camelcase - createdAt: created_at, // eslint-disable-line camelcase - updatedAt: updated_at, // eslint-disable-line camelcase - commentsCount: comments, - /* eslint-enable camelcase */ - createdBy: this.githubService.githubUserToAccountEntity(user), - }), - ); - } catch (error) { - this.loggerService.warn({ - message: `Failed to fetch contributions for ${owner}/${repository}: ${error}`, - meta: { owner, repository }, - }); - return []; - } - }), - ], - [], - ), - ) - ).reduce((pV, cV) => [...pV, ...cV], []); - if (filterFn) { - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore - contributions = contributions.filter(filterFn); - } - contributions = contributions.sort( - (a, b) => new Date(b.updatedAt).getTime() - new Date(a.updatedAt).getTime(), - ); + public async deleteAllButWithRunId(runId: string) { + return await this.sqliteService.db + .delete(contributionsTable) + .where(ne(contributionsTable.runId, runId)); + } - const filters: FilterDto[] = allFilterNames.map((name) => ({ name, options: [] })); + public async findForList() { + const statement = sql` + SELECT + p.id as id, + p.name as name, + json_group_array( + json_object('id', r.id, 'name', r.name, 'owner', r.owner, 'contributions', r.contributions) + ) AS repositories + FROM + (SELECT + r.id as id, + r.owner as owner, + r.name as name, + r.project_id as project_id, + json_group_array( + json_object('id', c.id, 'title', c.title, 'type', c.type, 'url', c.url, 'updated_at', c.updated_at, 'activity_count', c.activity_count) + ) AS contributions + FROM + ${contributionsTable} c + RIGHT JOIN + ${repositoriesTable} r ON c.id = r.project_id + GROUP BY + c.id) AS r + RIGHT JOIN + ${projectsTable} p ON r.project_id = p.id + `; - contributions.forEach(({ project, languages, labels }) => { - this.pushUniqueOption([{ name: project.slug, label: project.name }], filters[0].options); + const raw = this.sqliteService.db.all(statement); + const unStringifiedRaw = unStringifyDeep(raw); - this.pushUniqueOption( - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore - languages.map((language) => ({ name: language })), - filters[1].options, - ); + const reversed = reverseHierarchy(unStringifiedRaw, [ + { from: "repositories", setParentAs: "project" }, + { from: "contributions", setParentAs: "repository" }, + ]); - this.pushUniqueOption( - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore - labels.map((label) => ({ name: label })), - filters[2].options, - ); - }); + const camelCased = camelCaseObject(reversed); - return { - contributions, - filters, - }; + return camelCased; } - - private pushUniqueOption = (options: OptionDto[], filterOptions: OptionDto[]) => { - const uniqueOptions = options.filter( - (_option) => !filterOptions.some(({ name }) => _option.name === name), - ); - filterOptions.push(...uniqueOptions); - }; } diff --git a/api/src/contribution/table.ts b/api/src/contribution/table.ts new file mode 100644 index 000000000..52f4d5ad9 --- /dev/null +++ b/api/src/contribution/table.ts @@ -0,0 +1,25 @@ +import { Model } from "@dzcode.io/models/dist/_base"; +import { ContributionEntity } from "@dzcode.io/models/dist/contribution"; +import { sql } from "drizzle-orm"; +import { integer, sqliteTable, text } from "drizzle-orm/sqlite-core"; +import { repositoriesTable } from "src/repository/table"; + +export const contributionsTable = sqliteTable("contributions", { + id: integer("id", { mode: "number" }).primaryKey({ autoIncrement: true }), + recordImportedAt: text("record_imported_at") + .notNull() + .default(sql`CURRENT_TIMESTAMP`), + title: text("title").notNull(), + updatedAt: text("updated_at").notNull(), + url: text("url").notNull().unique(), + type: text("type").notNull().$type(), + runId: text("run_id").notNull(), + activityCount: integer("activity_count").notNull(), + repositoryId: integer("repository_id") + .notNull() + .references(() => repositoriesTable.id), +}); + +contributionsTable.$inferSelect satisfies Model; + +export type ContributionRow = typeof contributionsTable.$inferInsert; diff --git a/packages/models/src/contribution/__snapshots__/index.spec.ts.snap b/packages/models/src/contribution/__snapshots__/index.spec.ts.snap deleted file mode 100644 index 750d74aed..000000000 --- a/packages/models/src/contribution/__snapshots__/index.spec.ts.snap +++ /dev/null @@ -1,155 +0,0 @@ -// Jest Snapshot v1, https://goo.gl/fbAQLP - -exports[`should match snapshot when providing all fields: errors 1`] = `[]`; - -exports[`should match snapshot when providing all fields: output 1`] = ` -ContributionEntity { - "commentsCount": 0, - "createdAt": "2020-02-02T20:22:02.000Z", - "createdBy": AccountEntity { - "avatarUrl": "https://avatars.githubusercontent.com/u/20110076?v=4", - "id": "github/20110076", - "name": "Zakaria Mansouri", - "profileUrl": "/Account/github/20110076", - "username": "ZibanPirate", - }, - "id": "71", - "labels": [ - "discussion", - "good first issue", - ], - "languages": [ - "JavaScript", - "Shell", - ], - "project": ProjectReferenceEntity { - "name": "Leblad", - "slug": "Leblad", - }, - "title": "Update the data set", - "type": "issue", - "updatedAt": "2020-02-02T20:22:02.000Z", - "url": "https://github.com/dzcode-io/leblad/issues/71", -} -`; - -exports[`should match snapshot when providing required fields only: errors 1`] = `[]`; - -exports[`should match snapshot when providing required fields only: output 1`] = ` -ContributionEntity { - "commentsCount": 0, - "createdAt": "2020-02-02T20:22:02.000Z", - "createdBy": AccountEntity { - "avatarUrl": "https://avatars.githubusercontent.com/u/20110076?v=4", - "id": "github/20110076", - "name": "Zakaria Mansouri", - "profileUrl": "/Account/github/20110076", - "username": "ZibanPirate", - }, - "id": "71", - "labels": [ - "discussion", - "good first issue", - ], - "languages": [ - "JavaScript", - "Shell", - ], - "project": ProjectReferenceEntity { - "name": "Leblad", - "slug": "Leblad", - }, - "title": "Update the data set", - "type": "issue", - "updatedAt": "2020-02-02T20:22:02.000Z", - "url": "https://github.com/dzcode-io/leblad/issues/71", -} -`; - -exports[`should show an error that matches snapshot when passing empty object: errors 1`] = ` -[ - ValidationError { - "children": [], - "constraints": { - "isString": "id must be a string", - }, - "property": "id", - "target": ContributionEntity {}, - "value": undefined, - }, - ValidationError { - "children": [], - "constraints": { - "isString": "title must be a string", - }, - "property": "title", - "target": ContributionEntity {}, - "value": undefined, - }, - ValidationError { - "children": [], - "constraints": { - "isString": "type must be a string", - }, - "property": "type", - "target": ContributionEntity {}, - "value": undefined, - }, - ValidationError { - "children": [], - "constraints": { - "isUrl": "url must be an URL address", - }, - "property": "url", - "target": ContributionEntity {}, - "value": undefined, - }, - ValidationError { - "children": [], - "constraints": { - "isString": "each value in languages must be a string", - }, - "property": "languages", - "target": ContributionEntity {}, - "value": undefined, - }, - ValidationError { - "children": [], - "constraints": { - "isString": "each value in labels must be a string", - }, - "property": "labels", - "target": ContributionEntity {}, - "value": undefined, - }, - ValidationError { - "children": [], - "constraints": { - "isDateString": "createdAt must be a valid ISO 8601 date string", - }, - "property": "createdAt", - "target": ContributionEntity {}, - "value": undefined, - }, - ValidationError { - "children": [], - "constraints": { - "isDateString": "updatedAt must be a valid ISO 8601 date string", - }, - "property": "updatedAt", - "target": ContributionEntity {}, - "value": undefined, - }, - ValidationError { - "children": [], - "constraints": { - "isNumber": "commentsCount must be a number conforming to the specified constraints", - }, - "property": "commentsCount", - "target": ContributionEntity {}, - "value": undefined, - }, -] -`; - -exports[`should show an error that matches snapshot when passing empty object: output 1`] = `ContributionEntity {}`; diff --git a/packages/models/src/contribution/index.spec.ts b/packages/models/src/contribution/index.spec.ts deleted file mode 100644 index 3e90685be..000000000 --- a/packages/models/src/contribution/index.spec.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { runDTOTestCases } from "src/_test"; - -import { ContributionEntity } from "."; - -runDTOTestCases( - ContributionEntity, - { - activityCount: 0, - title: "Update the data set", - type: "ISSUE", - updatedAt: "2020-02-02T20:22:02.000Z", - url: "https://github.com/dzcode-io/leblad/issues/71", - id: 0, - runId: "test-run-id", - }, - {}, -); diff --git a/packages/models/src/contribution/index.ts b/packages/models/src/contribution/index.ts index 78602e086..7dbcb0de7 100644 --- a/packages/models/src/contribution/index.ts +++ b/packages/models/src/contribution/index.ts @@ -1,46 +1,9 @@ -import { Type } from "class-transformer"; -import { IsDateString, IsIn, IsNumber, IsString, IsUrl, ValidateNested } from "class-validator"; -import { BaseEntity, Model } from "src/_base"; -import { AccountEntity } from "src/account"; -import { ProjectReferenceEntity } from "src/project-reference"; - -export class ContributionEntityCompact extends BaseEntity { - // @TODO-ZM: move this to BaseEntity - @IsString() - id!: number; - - @IsString() - title!: string; - - @IsIn(["ISSUE", "PULL_REQUEST"]) - type!: "ISSUE" | "PULL_REQUEST"; - - @IsUrl() - url!: string; - - @IsDateString() - updatedAt!: string; - - @IsNumber() - activityCount!: number; -} -export class ContributionEntity extends ContributionEntityCompact { - @IsString() - runId!: string; -} - -export class ContributionEntityForList extends BaseEntity { - @ValidateNested() - @Type(() => AccountEntity) - createdBy!: Model; // Compact - - @ValidateNested() - @Type(() => ProjectReferenceEntity) - project!: Model; // Compact - - @IsString({ each: true }) - languages!: string[]; // Compact - - @IsString({ each: true }) - labels!: string[]; // Compact +export interface ContributionEntity { + id: number; + title: string; + type: "ISSUE" | "PULL_REQUEST"; + url: string; + updatedAt: string; + activityCount: number; + runId: string; } diff --git a/packages/models/src/project/index.ts b/packages/models/src/project/index.ts index fb2d2c401..744bb62a2 100644 --- a/packages/models/src/project/index.ts +++ b/packages/models/src/project/index.ts @@ -1,27 +1,7 @@ -import { Type } from "class-transformer"; -import { IsNumber, IsString, ValidateNested } from "class-validator"; -import { BaseEntity } from "src/_base"; -import { RepositoryEntityCompact } from "src/repository"; - -export class ProjectEntityCompact extends BaseEntity { +export interface ProjectEntity { // @TODO-ZM: move this to BaseEntity - @IsNumber() - id!: number; - - @IsString() - slug!: string; - - @IsString() - name!: string; -} - -export class ProjectEntity extends ProjectEntityCompact { - @IsString() - runId!: string; -} - -export class ProjectEntityForList extends ProjectEntityCompact { - @ValidateNested({ each: true }) - @Type(() => RepositoryEntityCompact) - repositories!: RepositoryEntityCompact[]; + id: number; + slug: string; + name: string; + runId: string; } diff --git a/packages/models/src/repository/__snapshots__/index.spec.ts.snap b/packages/models/src/repository/__snapshots__/index.spec.ts.snap deleted file mode 100644 index 72edf3836..000000000 --- a/packages/models/src/repository/__snapshots__/index.spec.ts.snap +++ /dev/null @@ -1,64 +0,0 @@ -// Jest Snapshot v1, https://goo.gl/fbAQLP - -exports[`should match snapshot when providing all fields: errors 1`] = `[]`; - -exports[`should match snapshot when providing all fields: output 1`] = ` -RepositoryEntity { - "contributions": [], - "contributors": [], - "owner": "dzcode-io", - "provider": "github", - "repository": "leblad", - "stats": RepositoryStatsEntity { - "contributionCount": 10, - "languages": [ - "TypeScript", - "Rust", - ], - }, -} -`; - -exports[`should match snapshot when providing required fields only: errors 1`] = `[]`; - -exports[`should match snapshot when providing required fields only: output 1`] = ` -RepositoryEntity { - "owner": "dzcode-io", - "provider": "github", - "repository": "leblad", -} -`; - -exports[`should show an error that matches snapshot when passing empty object: errors 1`] = ` -[ - ValidationError { - "children": [], - "constraints": { - "isIn": "provider must be one of the following values: github, gitlab", - }, - "property": "provider", - "target": RepositoryEntity {}, - "value": undefined, - }, - ValidationError { - "children": [], - "constraints": { - "isString": "owner must be a string", - }, - "property": "owner", - "target": RepositoryEntity {}, - "value": undefined, - }, - ValidationError { - "children": [], - "constraints": { - "isString": "repository must be a string", - }, - "property": "repository", - "target": RepositoryEntity {}, - "value": undefined, - }, -] -`; - -exports[`should show an error that matches snapshot when passing empty object: output 1`] = `RepositoryEntity {}`; diff --git a/packages/models/src/repository/index.spec.ts b/packages/models/src/repository/index.spec.ts deleted file mode 100644 index 24cb83cc2..000000000 --- a/packages/models/src/repository/index.spec.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { runDTOTestCases } from "src/_test"; - -import { RepositoryEntity } from "."; - -runDTOTestCases( - RepositoryEntity, - { - provider: "github", - owner: "dzcode-io", - name: "leblad", - id: 0, - runId: "initial-run-id", - }, - { - contributions: [], - contributors: [], - }, -); diff --git a/packages/models/src/repository/index.ts b/packages/models/src/repository/index.ts index be27e6895..6fd42de19 100644 --- a/packages/models/src/repository/index.ts +++ b/packages/models/src/repository/index.ts @@ -1,43 +1,11 @@ -import { Type } from "class-transformer"; -import { IsIn, IsNumber, IsString, ValidateNested } from "class-validator"; -import { BaseEntity } from "src/_base"; -import { AccountEntity } from "src/account"; -import { ContributionEntity } from "src/contribution"; - const RepositoryProviders = ["github", "gitlab"] as const; type RepositoryProvider = (typeof RepositoryProviders)[number]; -export class RepositoryEntityCompact extends BaseEntity { +export interface RepositoryEntity { // @TODO-ZM: move this to BaseEntity - @IsNumber() - id!: number; - - @IsString() - owner!: string; - - @IsString() - name!: string; -} - -export class RepositoryEntity extends RepositoryEntityCompact { - @IsString() - runId!: string; - - @IsIn(RepositoryProviders) - provider!: RepositoryProvider; -} - -export class RepositoryEntityForList extends RepositoryEntity { - // TODO-ZM: add programming languages - // @ValidateNested({ each: true }) - // @Type(() => ProgrammingLanguageEntity) - // programmingLanguages!: ProgrammingLanguageEntityCompact[]; - - @ValidateNested({ each: true }) - @Type(() => AccountEntity) - contributors!: AccountEntity[]; - - @ValidateNested({ each: true }) - @Type(() => ContributionEntity) - contributions!: ContributionEntity[]; + id: number; + owner: string; + name: string; + runId: string; + provider: RepositoryProvider; } From cdcbf7b800c4e6876de191c65dd8ebfbebe6ce77 Mon Sep 17 00:00:00 2001 From: Zakaria Mansouri Date: Fri, 6 Sep 2024 21:27:04 +0200 Subject: [PATCH 03/12] refactored projects and contribs endpoints --- api/src/contribution/controller.ts | 26 ++++---------------------- api/src/contribution/types.ts | 19 ++++++++++--------- api/src/project/controller.ts | 5 ----- api/src/project/repository.ts | 12 ++++++------ api/src/project/types.ts | 12 +++--------- web/src/pages/contribute/index.tsx | 27 ++++++++++++++++----------- 6 files changed, 39 insertions(+), 62 deletions(-) diff --git a/api/src/contribution/controller.ts b/api/src/contribution/controller.ts index 7bfbea048..be91fd1d1 100644 --- a/api/src/contribution/controller.ts +++ b/api/src/contribution/controller.ts @@ -1,9 +1,8 @@ -import { Controller, Get, QueryParams } from "routing-controllers"; -import { OpenAPI, ResponseSchema } from "routing-controllers-openapi"; +import { Controller, Get } from "routing-controllers"; import { Service } from "typedi"; import { ContributionRepository } from "./repository"; -import { GetContributionsQueryDto, GetContributionsResponseDto } from "./types"; +import { GetContributionsResponseDto } from "./types"; @Service() @Controller("/Contributions") @@ -11,28 +10,11 @@ export class ContributionController { constructor(private readonly contributionRepository: ContributionRepository) {} @Get("/") - @OpenAPI({ - summary: "Return a list of contributions for all projects listed in dzcode.io", - }) - @ResponseSchema(GetContributionsResponseDto) - public async getContributions( - @QueryParams() { labels, languages, projects }: GetContributionsQueryDto, - ): Promise { - const { contributions, filters } = await this.contributionRepository.find( - (contribution) => - !contribution.createdBy.username.includes("[bot]") && - (labels.length === 0 || labels.some((label) => contribution.labels.includes(label))) && - (languages.length === 0 || - languages.some((language) => contribution.languages.includes(language))) && - (projects.length === 0 || - projects.some((project) => { - return contribution.project.slug === project; - })), - ); + public async getContributions(): Promise { + const contributions = await this.contributionRepository.findForList(); return { contributions, - filters, }; } } diff --git a/api/src/contribution/types.ts b/api/src/contribution/types.ts index 77c154aaf..c6562ccc8 100644 --- a/api/src/contribution/types.ts +++ b/api/src/contribution/types.ts @@ -1,5 +1,6 @@ -import { Model } from "@dzcode.io/models/dist/_base"; import { ContributionEntity } from "@dzcode.io/models/dist/contribution"; +import { ProjectEntity } from "@dzcode.io/models/dist/project"; +import { RepositoryEntity } from "@dzcode.io/models/dist/repository"; import { Transform, TransformFnParams, Type } from "class-transformer"; import { IsBoolean, IsIn, IsOptional, IsString, ValidateNested } from "class-validator"; import { GeneralResponseDto } from "src/app/types"; @@ -28,14 +29,14 @@ export class FilterDto { options!: OptionDto[]; } -export class GetContributionsResponseDto extends GeneralResponseDto { - @ValidateNested({ each: true }) - @Type(() => ContributionEntity) - contributions!: Model[]; - - @ValidateNested({ each: true }) - @Type(() => FilterDto) - filters!: FilterDto[]; +export interface GetContributionsResponseDto extends GeneralResponseDto { + contributions: Array< + Pick & { + repository: Pick & { + project: Pick; + }; + } + >; } const transformFilterOptions = ({ value }: TransformFnParams) => { diff --git a/api/src/project/controller.ts b/api/src/project/controller.ts index 16daf1a5c..a0fde7753 100644 --- a/api/src/project/controller.ts +++ b/api/src/project/controller.ts @@ -1,5 +1,4 @@ import { Controller, Get } from "routing-controllers"; -import { OpenAPI, ResponseSchema } from "routing-controllers-openapi"; import { Service } from "typedi"; import { ProjectRepository } from "./repository"; @@ -11,10 +10,6 @@ export class ProjectController { constructor(private readonly projectRepository: ProjectRepository) {} @Get("/") - @OpenAPI({ - summary: "Return all projects listed in dzcode.io website", - }) - @ResponseSchema(GetProjectsResponseDto) public async getProjects(): Promise { const projects = await this.projectRepository.findForList(); diff --git a/api/src/project/repository.ts b/api/src/project/repository.ts index ffb6da8ed..272ee40bd 100644 --- a/api/src/project/repository.ts +++ b/api/src/project/repository.ts @@ -1,6 +1,5 @@ -import { ProjectEntityForList } from "@dzcode.io/models/dist/project"; +import { ProjectEntity } from "@dzcode.io/models/dist/project"; import { ne, sql } from "drizzle-orm"; -import { validatePlainObject } from "src/_utils/validator/validate-plain-object"; import { repositoriesTable } from "src/repository/table"; import { SQLiteService } from "src/sqlite/service"; import { Service } from "typedi"; @@ -12,6 +11,7 @@ export class ProjectRepository { constructor(private readonly sqliteService: SQLiteService) {} public async findForList() { + // @TODO-ZM: reverse hierarchy instead here const statement = sql` SELECT p.id as id, @@ -29,12 +29,12 @@ export class ProjectRepository { `; const raw = this.sqliteService.db.all(statement) as Array< // the SQL query above returns a stringified JSON for the `repositories` column - Omit & { repositories: string } + Omit & { repositories: string } >; - const projectsForList: ProjectEntityForList[] = raw.map((row) => { + const projectsForList: ProjectEntity[] = raw.map((row) => { const notYetValid = { ...row, repositories: JSON.parse(row.repositories) }; - const validated = validatePlainObject(ProjectEntityForList, notYetValid, true); - return validated; + + return notYetValid; }); return projectsForList; diff --git a/api/src/project/types.ts b/api/src/project/types.ts index 426fb6a38..cf2cc84fa 100644 --- a/api/src/project/types.ts +++ b/api/src/project/types.ts @@ -1,12 +1,6 @@ -import { ProjectEntityForList } from "@dzcode.io/models/dist/project"; -import { Type } from "class-transformer"; -import { ValidateNested } from "class-validator"; +import { ProjectEntity } from "@dzcode.io/models/dist/project"; import { GeneralResponseDto } from "src/app/types"; -// @TODO-ZM: remove Model<> from existence - -export class GetProjectsResponseDto extends GeneralResponseDto { - @ValidateNested({ each: true }) - @Type(() => ProjectEntityForList) - projects!: Array; +export interface GetProjectsResponseDto extends GeneralResponseDto { + projects: ProjectEntity[]; } diff --git a/web/src/pages/contribute/index.tsx b/web/src/pages/contribute/index.tsx index fa7aa6a95..ed7762943 100644 --- a/web/src/pages/contribute/index.tsx +++ b/web/src/pages/contribute/index.tsx @@ -54,8 +54,13 @@ export default function Page(): JSX.Element {

- {contribution.project.name} -
+ + {contribution.repository.project.name} + + + {contribution.repository.owner}/{contribution.repository.name} + + {/*
{contribution.labels.map((label, labelIndex) => ( {label} @@ -66,17 +71,14 @@ export default function Page(): JSX.Element { {language} ))} -
+
*/}
- + /> */}
-
- {getElapsedTime(contribution.updatedAt, localize("elapsed-time-suffixes"))} -
- {contribution.commentsCount > 0 && ( + {contribution.activityCount > 0 && (
- {contribution.commentsCount} + {contribution.activityCount}
)} +
+ {getElapsedTime(contribution.updatedAt, localize("elapsed-time-suffixes"))} +
- {contribution.type === "issue" + {contribution.type === "ISSUE" ? localize("contribute-read-issue") : localize("contribute-review-changes")} From 17e0707ad69e15596b0f868ada7e5c7168290108 Mon Sep 17 00:00:00 2001 From: Zakaria Mansouri Date: Fri, 6 Sep 2024 22:38:49 +0200 Subject: [PATCH 04/12] added contributors to database --- api/db/migrations/0000_new_lake.sql | 21 -- api/db/migrations/0000_sudden_doctor_doom.sql | 48 ++++ api/db/migrations/0001_warm_lockheed.sql | 14 -- api/db/migrations/0002_mighty_may_parker.sql | 2 - api/db/migrations/meta/0000_snapshot.json | 173 ++++++++++++- api/db/migrations/meta/0001_snapshot.json | 234 ------------------ api/db/migrations/meta/0002_snapshot.json | 234 ------------------ api/db/migrations/meta/_journal.json | 18 +- api/src/contribution/table.ts | 4 + api/src/contributor/repository.ts | 27 ++ api/src/contributor/table.ts | 20 ++ api/src/digest/cron.ts | 17 +- packages/models/src/contributor/index.ts | 9 + .../project/__snapshots__/index.spec.ts.snap | 68 ----- 14 files changed, 298 insertions(+), 591 deletions(-) delete mode 100644 api/db/migrations/0000_new_lake.sql create mode 100644 api/db/migrations/0000_sudden_doctor_doom.sql delete mode 100644 api/db/migrations/0001_warm_lockheed.sql delete mode 100644 api/db/migrations/0002_mighty_may_parker.sql delete mode 100644 api/db/migrations/meta/0001_snapshot.json delete mode 100644 api/db/migrations/meta/0002_snapshot.json create mode 100644 api/src/contributor/repository.ts create mode 100644 api/src/contributor/table.ts create mode 100644 packages/models/src/contributor/index.ts delete mode 100644 packages/models/src/project/__snapshots__/index.spec.ts.snap diff --git a/api/db/migrations/0000_new_lake.sql b/api/db/migrations/0000_new_lake.sql deleted file mode 100644 index caf956a56..000000000 --- a/api/db/migrations/0000_new_lake.sql +++ /dev/null @@ -1,21 +0,0 @@ -CREATE TABLE `projects` ( - `id` integer PRIMARY KEY AUTOINCREMENT NOT NULL, - `record_imported_at` text DEFAULT CURRENT_TIMESTAMP NOT NULL, - `name` text NOT NULL, - `slug` text NOT NULL, - `run_id` text DEFAULT 'initial-run-id' NOT NULL -); ---> statement-breakpoint -CREATE TABLE `repositories` ( - `id` integer PRIMARY KEY AUTOINCREMENT NOT NULL, - `record_imported_at` text DEFAULT CURRENT_TIMESTAMP NOT NULL, - `provider` text NOT NULL, - `owner` text NOT NULL, - `name` text NOT NULL, - `run_id` text DEFAULT 'initial-run-id' NOT NULL, - `project_id` integer NOT NULL, - FOREIGN KEY (`project_id`) REFERENCES `projects`(`id`) ON UPDATE no action ON DELETE no action -); ---> statement-breakpoint -CREATE UNIQUE INDEX `projects_slug_unique` ON `projects` (`slug`);--> statement-breakpoint -CREATE UNIQUE INDEX `repositories_provider_owner_name_unique` ON `repositories` (`provider`,`owner`,`name`); \ No newline at end of file diff --git a/api/db/migrations/0000_sudden_doctor_doom.sql b/api/db/migrations/0000_sudden_doctor_doom.sql new file mode 100644 index 000000000..9bc405bec --- /dev/null +++ b/api/db/migrations/0000_sudden_doctor_doom.sql @@ -0,0 +1,48 @@ +CREATE TABLE `contributions` ( + `id` integer PRIMARY KEY AUTOINCREMENT NOT NULL, + `record_imported_at` text DEFAULT CURRENT_TIMESTAMP NOT NULL, + `title` text NOT NULL, + `updated_at` text NOT NULL, + `url` text NOT NULL, + `type` text NOT NULL, + `run_id` text NOT NULL, + `activity_count` integer NOT NULL, + `repository_id` integer NOT NULL, + `contributor_id` integer NOT NULL, + FOREIGN KEY (`repository_id`) REFERENCES `repositories`(`id`) ON UPDATE no action ON DELETE no action, + FOREIGN KEY (`contributor_id`) REFERENCES `contributors`(`id`) ON UPDATE no action ON DELETE no action +); +--> statement-breakpoint +CREATE TABLE `contributors` ( + `id` integer PRIMARY KEY AUTOINCREMENT NOT NULL, + `record_imported_at` text DEFAULT CURRENT_TIMESTAMP NOT NULL, + `run_id` text DEFAULT 'initial-run-id' NOT NULL, + `name` text NOT NULL, + `username` text NOT NULL, + `url` text NOT NULL, + `avatar_url` text NOT NULL +); +--> statement-breakpoint +CREATE TABLE `projects` ( + `id` integer PRIMARY KEY AUTOINCREMENT NOT NULL, + `record_imported_at` text DEFAULT CURRENT_TIMESTAMP NOT NULL, + `name` text NOT NULL, + `slug` text NOT NULL, + `run_id` text DEFAULT 'initial-run-id' NOT NULL +); +--> statement-breakpoint +CREATE TABLE `repositories` ( + `id` integer PRIMARY KEY AUTOINCREMENT NOT NULL, + `record_imported_at` text DEFAULT CURRENT_TIMESTAMP NOT NULL, + `provider` text NOT NULL, + `owner` text NOT NULL, + `name` text NOT NULL, + `run_id` text DEFAULT 'initial-run-id' NOT NULL, + `project_id` integer NOT NULL, + FOREIGN KEY (`project_id`) REFERENCES `projects`(`id`) ON UPDATE no action ON DELETE no action +); +--> statement-breakpoint +CREATE UNIQUE INDEX `contributions_url_unique` ON `contributions` (`url`);--> statement-breakpoint +CREATE UNIQUE INDEX `contributors_url_unique` ON `contributors` (`url`);--> statement-breakpoint +CREATE UNIQUE INDEX `projects_slug_unique` ON `projects` (`slug`);--> statement-breakpoint +CREATE UNIQUE INDEX `repositories_provider_owner_name_unique` ON `repositories` (`provider`,`owner`,`name`); \ No newline at end of file diff --git a/api/db/migrations/0001_warm_lockheed.sql b/api/db/migrations/0001_warm_lockheed.sql deleted file mode 100644 index 9df3ca887..000000000 --- a/api/db/migrations/0001_warm_lockheed.sql +++ /dev/null @@ -1,14 +0,0 @@ -CREATE TABLE `contributions` ( - `id` integer PRIMARY KEY AUTOINCREMENT NOT NULL, - `record_imported_at` text DEFAULT CURRENT_TIMESTAMP NOT NULL, - `name` text NOT NULL, - `updated_at` text NOT NULL, - `url` text NOT NULL, - `type` text NOT NULL, - `run_id` text NOT NULL, - `activity_count` integer NOT NULL, - `repository_id` integer NOT NULL, - FOREIGN KEY (`repository_id`) REFERENCES `repositories`(`id`) ON UPDATE no action ON DELETE no action -); ---> statement-breakpoint -CREATE UNIQUE INDEX `contributions_url_unique` ON `contributions` (`url`); \ No newline at end of file diff --git a/api/db/migrations/0002_mighty_may_parker.sql b/api/db/migrations/0002_mighty_may_parker.sql deleted file mode 100644 index 7095e1b0f..000000000 --- a/api/db/migrations/0002_mighty_may_parker.sql +++ /dev/null @@ -1,2 +0,0 @@ -ALTER TABLE `contributions` ADD `title` text NOT NULL;--> statement-breakpoint -ALTER TABLE `contributions` DROP COLUMN `name`; \ No newline at end of file diff --git a/api/db/migrations/meta/0000_snapshot.json b/api/db/migrations/meta/0000_snapshot.json index 743b04b73..04bb43a03 100644 --- a/api/db/migrations/meta/0000_snapshot.json +++ b/api/db/migrations/meta/0000_snapshot.json @@ -1,9 +1,180 @@ { "version": "6", "dialect": "sqlite", - "id": "ab1b512a-8ca8-44e1-b497-445465117250", + "id": "ba41012f-4495-42ff-ace1-61bd7eaef476", "prevId": "00000000-0000-0000-0000-000000000000", "tables": { + "contributions": { + "name": "contributions", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "record_imported_at": { + "name": "record_imported_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "activity_count": { + "name": "activity_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "repository_id": { + "name": "repository_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "contributor_id": { + "name": "contributor_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "contributions_url_unique": { + "name": "contributions_url_unique", + "columns": ["url"], + "isUnique": true + } + }, + "foreignKeys": { + "contributions_repository_id_repositories_id_fk": { + "name": "contributions_repository_id_repositories_id_fk", + "tableFrom": "contributions", + "tableTo": "repositories", + "columnsFrom": ["repository_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "contributions_contributor_id_contributors_id_fk": { + "name": "contributions_contributor_id_contributors_id_fk", + "tableFrom": "contributions", + "tableTo": "contributors", + "columnsFrom": ["contributor_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "contributors": { + "name": "contributors", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "record_imported_at": { + "name": "record_imported_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + }, + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'initial-run-id'" + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "username": { + "name": "username", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "avatar_url": { + "name": "avatar_url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "contributors_url_unique": { + "name": "contributors_url_unique", + "columns": ["url"], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, "projects": { "name": "projects", "columns": { diff --git a/api/db/migrations/meta/0001_snapshot.json b/api/db/migrations/meta/0001_snapshot.json deleted file mode 100644 index 7930c5c7e..000000000 --- a/api/db/migrations/meta/0001_snapshot.json +++ /dev/null @@ -1,234 +0,0 @@ -{ - "version": "6", - "dialect": "sqlite", - "id": "d333591a-11be-4b30-89f8-4da0aa5bd906", - "prevId": "ab1b512a-8ca8-44e1-b497-445465117250", - "tables": { - "contributions": { - "name": "contributions", - "columns": { - "id": { - "name": "id", - "type": "integer", - "primaryKey": true, - "notNull": true, - "autoincrement": true - }, - "record_imported_at": { - "name": "record_imported_at", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "CURRENT_TIMESTAMP" - }, - "name": { - "name": "name", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "updated_at": { - "name": "updated_at", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "url": { - "name": "url", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "type": { - "name": "type", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "run_id": { - "name": "run_id", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "activity_count": { - "name": "activity_count", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "repository_id": { - "name": "repository_id", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - } - }, - "indexes": { - "contributions_url_unique": { - "name": "contributions_url_unique", - "columns": ["url"], - "isUnique": true - } - }, - "foreignKeys": { - "contributions_repository_id_repositories_id_fk": { - "name": "contributions_repository_id_repositories_id_fk", - "tableFrom": "contributions", - "tableTo": "repositories", - "columnsFrom": ["repository_id"], - "columnsTo": ["id"], - "onDelete": "no action", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": {} - }, - "projects": { - "name": "projects", - "columns": { - "id": { - "name": "id", - "type": "integer", - "primaryKey": true, - "notNull": true, - "autoincrement": true - }, - "record_imported_at": { - "name": "record_imported_at", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "CURRENT_TIMESTAMP" - }, - "name": { - "name": "name", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "slug": { - "name": "slug", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "run_id": { - "name": "run_id", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'initial-run-id'" - } - }, - "indexes": { - "projects_slug_unique": { - "name": "projects_slug_unique", - "columns": ["slug"], - "isUnique": true - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": {}, - "uniqueConstraints": {} - }, - "repositories": { - "name": "repositories", - "columns": { - "id": { - "name": "id", - "type": "integer", - "primaryKey": true, - "notNull": true, - "autoincrement": true - }, - "record_imported_at": { - "name": "record_imported_at", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "CURRENT_TIMESTAMP" - }, - "provider": { - "name": "provider", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "owner": { - "name": "owner", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "name": { - "name": "name", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "run_id": { - "name": "run_id", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'initial-run-id'" - }, - "project_id": { - "name": "project_id", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - } - }, - "indexes": { - "repositories_provider_owner_name_unique": { - "name": "repositories_provider_owner_name_unique", - "columns": ["provider", "owner", "name"], - "isUnique": true - } - }, - "foreignKeys": { - "repositories_project_id_projects_id_fk": { - "name": "repositories_project_id_projects_id_fk", - "tableFrom": "repositories", - "tableTo": "projects", - "columnsFrom": ["project_id"], - "columnsTo": ["id"], - "onDelete": "no action", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": {} - } - }, - "enums": {}, - "_meta": { - "schemas": {}, - "tables": {}, - "columns": {} - }, - "internal": { - "indexes": {} - } -} diff --git a/api/db/migrations/meta/0002_snapshot.json b/api/db/migrations/meta/0002_snapshot.json deleted file mode 100644 index 29b79923b..000000000 --- a/api/db/migrations/meta/0002_snapshot.json +++ /dev/null @@ -1,234 +0,0 @@ -{ - "version": "6", - "dialect": "sqlite", - "id": "4c5b2a9b-5f9e-496c-a73e-e260a0513b86", - "prevId": "d333591a-11be-4b30-89f8-4da0aa5bd906", - "tables": { - "contributions": { - "name": "contributions", - "columns": { - "id": { - "name": "id", - "type": "integer", - "primaryKey": true, - "notNull": true, - "autoincrement": true - }, - "record_imported_at": { - "name": "record_imported_at", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "CURRENT_TIMESTAMP" - }, - "title": { - "name": "title", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "updated_at": { - "name": "updated_at", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "url": { - "name": "url", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "type": { - "name": "type", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "run_id": { - "name": "run_id", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "activity_count": { - "name": "activity_count", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "repository_id": { - "name": "repository_id", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - } - }, - "indexes": { - "contributions_url_unique": { - "name": "contributions_url_unique", - "columns": ["url"], - "isUnique": true - } - }, - "foreignKeys": { - "contributions_repository_id_repositories_id_fk": { - "name": "contributions_repository_id_repositories_id_fk", - "tableFrom": "contributions", - "tableTo": "repositories", - "columnsFrom": ["repository_id"], - "columnsTo": ["id"], - "onDelete": "no action", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": {} - }, - "projects": { - "name": "projects", - "columns": { - "id": { - "name": "id", - "type": "integer", - "primaryKey": true, - "notNull": true, - "autoincrement": true - }, - "record_imported_at": { - "name": "record_imported_at", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "CURRENT_TIMESTAMP" - }, - "name": { - "name": "name", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "slug": { - "name": "slug", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "run_id": { - "name": "run_id", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'initial-run-id'" - } - }, - "indexes": { - "projects_slug_unique": { - "name": "projects_slug_unique", - "columns": ["slug"], - "isUnique": true - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": {}, - "uniqueConstraints": {} - }, - "repositories": { - "name": "repositories", - "columns": { - "id": { - "name": "id", - "type": "integer", - "primaryKey": true, - "notNull": true, - "autoincrement": true - }, - "record_imported_at": { - "name": "record_imported_at", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "CURRENT_TIMESTAMP" - }, - "provider": { - "name": "provider", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "owner": { - "name": "owner", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "name": { - "name": "name", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "run_id": { - "name": "run_id", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'initial-run-id'" - }, - "project_id": { - "name": "project_id", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - } - }, - "indexes": { - "repositories_provider_owner_name_unique": { - "name": "repositories_provider_owner_name_unique", - "columns": ["provider", "owner", "name"], - "isUnique": true - } - }, - "foreignKeys": { - "repositories_project_id_projects_id_fk": { - "name": "repositories_project_id_projects_id_fk", - "tableFrom": "repositories", - "tableTo": "projects", - "columnsFrom": ["project_id"], - "columnsTo": ["id"], - "onDelete": "no action", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": {} - } - }, - "enums": {}, - "_meta": { - "schemas": {}, - "tables": {}, - "columns": {} - }, - "internal": { - "indexes": {} - } -} diff --git a/api/db/migrations/meta/_journal.json b/api/db/migrations/meta/_journal.json index a3a5cb484..434d06b88 100644 --- a/api/db/migrations/meta/_journal.json +++ b/api/db/migrations/meta/_journal.json @@ -5,22 +5,8 @@ { "idx": 0, "version": "6", - "when": 1725462108723, - "tag": "0000_new_lake", - "breakpoints": true - }, - { - "idx": 1, - "version": "6", - "when": 1725564569182, - "tag": "0001_warm_lockheed", - "breakpoints": true - }, - { - "idx": 2, - "version": "6", - "when": 1725568393657, - "tag": "0002_mighty_may_parker", + "when": 1725654548149, + "tag": "0000_sudden_doctor_doom", "breakpoints": true } ] diff --git a/api/src/contribution/table.ts b/api/src/contribution/table.ts index 52f4d5ad9..03d715cfe 100644 --- a/api/src/contribution/table.ts +++ b/api/src/contribution/table.ts @@ -2,6 +2,7 @@ import { Model } from "@dzcode.io/models/dist/_base"; import { ContributionEntity } from "@dzcode.io/models/dist/contribution"; import { sql } from "drizzle-orm"; import { integer, sqliteTable, text } from "drizzle-orm/sqlite-core"; +import { contributorsTable } from "src/contributor/table"; import { repositoriesTable } from "src/repository/table"; export const contributionsTable = sqliteTable("contributions", { @@ -18,6 +19,9 @@ export const contributionsTable = sqliteTable("contributions", { repositoryId: integer("repository_id") .notNull() .references(() => repositoriesTable.id), + contributorId: integer("contributor_id") + .notNull() + .references(() => contributorsTable.id), }); contributionsTable.$inferSelect satisfies Model; diff --git a/api/src/contributor/repository.ts b/api/src/contributor/repository.ts new file mode 100644 index 000000000..45ae3d382 --- /dev/null +++ b/api/src/contributor/repository.ts @@ -0,0 +1,27 @@ +import { ne } from "drizzle-orm"; +import { SQLiteService } from "src/sqlite/service"; +import { Service } from "typedi"; + +import { ContributorRow, contributorsTable } from "./table"; + +@Service() +export class ContributorRepository { + constructor(private readonly sqliteService: SQLiteService) {} + + public async upsert(contributor: ContributorRow) { + return await this.sqliteService.db + .insert(contributorsTable) + .values(contributor) + .onConflictDoUpdate({ + target: contributorsTable.url, + set: contributor, + }) + .returning({ id: contributorsTable.id }); + } + + public async deleteAllButWithRunId(runId: string) { + return await this.sqliteService.db + .delete(contributorsTable) + .where(ne(contributorsTable.runId, runId)); + } +} diff --git a/api/src/contributor/table.ts b/api/src/contributor/table.ts new file mode 100644 index 000000000..72be1d5e1 --- /dev/null +++ b/api/src/contributor/table.ts @@ -0,0 +1,20 @@ +import { Model } from "@dzcode.io/models/dist/_base"; +import { ContributorEntity } from "@dzcode.io/models/dist/contributor"; +import { sql } from "drizzle-orm"; +import { integer, sqliteTable, text } from "drizzle-orm/sqlite-core"; + +export const contributorsTable = sqliteTable("contributors", { + id: integer("id", { mode: "number" }).primaryKey({ autoIncrement: true }), + recordImportedAt: text("record_imported_at") + .notNull() + .default(sql`CURRENT_TIMESTAMP`), + runId: text("run_id").notNull().default("initial-run-id"), + name: text("name").notNull(), + username: text("username").notNull(), + url: text("url").notNull().unique(), + avatarUrl: text("avatar_url").notNull(), +}); + +contributorsTable.$inferSelect satisfies Model; + +export type ContributorRow = typeof contributorsTable.$inferInsert; diff --git a/api/src/digest/cron.ts b/api/src/digest/cron.ts index 646dfea98..cda12c1c8 100644 --- a/api/src/digest/cron.ts +++ b/api/src/digest/cron.ts @@ -1,6 +1,7 @@ import { captureException, cron } from "@sentry/node"; import { CronJob } from "cron"; import { ContributionRepository } from "src/contribution/repository"; +import { ContributorRepository } from "src/contributor/repository"; import { DataService } from "src/data/service"; import { GithubService } from "src/github/service"; import { LoggerService } from "src/logger/service"; @@ -20,6 +21,7 @@ export class DigestCron { private readonly projectsRepository: ProjectRepository, private readonly repositoriesRepository: RepositoryRepository, private readonly contributionsRepository: ContributionRepository, + private readonly contributorsRepository: ContributorRepository, ) { const SentryCronJob = cron.instrumentCron(CronJob, "DigestCron"); new SentryCronJob( @@ -64,6 +66,8 @@ export class DigestCron { const projectsFromDataFolder = await this.dataService.listProjects(); + // @TODO-ZM: add data with recordStatus="draft", delete, then update to recordStatus="ok" + // @TODO-ZM: in all repos, filter by recordStatus="ok" for (const project of projectsFromDataFolder) { const [{ id: projectId }] = await this.projectsRepository.upsert({ ...project, runId }); @@ -90,6 +94,15 @@ export class DigestCron { }); for (const issue of issues.issues) { + const githubUser = await this.githubService.getUser({ username: issue.user.login }); + const [{ id: contributorId }] = await this.contributorsRepository.upsert({ + name: githubUser.name || githubUser.login, + username: githubUser.login, + url: githubUser.html_url, + avatarUrl: githubUser.avatar_url, + runId, + }); + const type = issue.pull_request ? "PULL_REQUEST" : "ISSUE"; const [{ id: contributionId }] = await this.contributionsRepository.upsert({ title: issue.title, @@ -97,8 +110,9 @@ export class DigestCron { updatedAt: issue.updated_at, activityCount: issue.comments, runId, - repositoryId, url: type === "PULL_REQUEST" ? issue.pull_request.html_url : issue.html_url, + repositoryId, + contributorId, }); console.log("contributionId", contributionId); @@ -114,6 +128,7 @@ export class DigestCron { } } + await this.contributorsRepository.deleteAllButWithRunId(runId); await this.contributionsRepository.deleteAllButWithRunId(runId); await this.repositoriesRepository.deleteAllButWithRunId(runId); await this.projectsRepository.deleteAllButWithRunId(runId); diff --git a/packages/models/src/contributor/index.ts b/packages/models/src/contributor/index.ts new file mode 100644 index 000000000..08bce9e5b --- /dev/null +++ b/packages/models/src/contributor/index.ts @@ -0,0 +1,9 @@ +export interface ContributorEntity { + // @TODO-ZM: move this to BaseEntity + id: number; + runId: string; + name: string; + username: string; + url: string; + avatarUrl: string; +} diff --git a/packages/models/src/project/__snapshots__/index.spec.ts.snap b/packages/models/src/project/__snapshots__/index.spec.ts.snap deleted file mode 100644 index 4cbb6de2b..000000000 --- a/packages/models/src/project/__snapshots__/index.spec.ts.snap +++ /dev/null @@ -1,68 +0,0 @@ -// Jest Snapshot v1, https://goo.gl/fbAQLP - -exports[`should match snapshot when providing all fields: errors 1`] = `[]`; - -exports[`should match snapshot when providing all fields: output 1`] = ` -ProjectEntity { - "name": "Leblad", - "repositories": [ - RepositoryEntity { - "owner": "dzcode-io", - "provider": "github", - "repository": "leblad", - }, - RepositoryEntity { - "owner": "abderrahmaneMustapha", - "provider": "github", - "repository": "leblad-py", - }, - ], - "slug": "Leblad", -} -`; - -exports[`should match snapshot when providing required fields only: errors 1`] = `[]`; - -exports[`should match snapshot when providing required fields only: output 1`] = ` -ProjectEntity { - "name": "Leblad", - "repositories": [ - RepositoryEntity { - "owner": "dzcode-io", - "provider": "github", - "repository": "leblad", - }, - RepositoryEntity { - "owner": "abderrahmaneMustapha", - "provider": "github", - "repository": "leblad-py", - }, - ], - "slug": "Leblad", -} -`; - -exports[`should show an error that matches snapshot when passing empty object: errors 1`] = ` -[ - ValidationError { - "children": [], - "constraints": { - "isString": "slug must be a string", - }, - "property": "slug", - "target": ProjectEntity {}, - "value": undefined, - }, - ValidationError { - "children": [], - "constraints": { - "isString": "name must be a string", - }, - "property": "name", - "target": ProjectEntity {}, - "value": undefined, - }, -] -`; - -exports[`should show an error that matches snapshot when passing empty object: output 1`] = `ProjectEntity {}`; From fbfc9a377d2ef2ce845e4fb2bffc37a306b1c89c Mon Sep 17 00:00:00 2001 From: Zakaria Mansouri Date: Sat, 7 Sep 2024 19:40:21 +0200 Subject: [PATCH 05/12] populate contribution with project --- api/src/_utils/case.ts | 21 +++++++++++---- api/src/contribution/repository.ts | 43 ++++++++++++++++++++++++++---- api/src/contribution/types.ts | 2 ++ api/src/digest/cron.ts | 9 +++++++ api/src/project/repository.ts | 6 ++++- web/src/pages/contribute/index.tsx | 13 +++++---- 6 files changed, 76 insertions(+), 18 deletions(-) diff --git a/api/src/_utils/case.ts b/api/src/_utils/case.ts index 9870d38ad..709c38e00 100644 --- a/api/src/_utils/case.ts +++ b/api/src/_utils/case.ts @@ -2,10 +2,21 @@ import camelCase from "lodash/camelCase"; import mapKeys from "lodash/mapKeys"; export function camelCaseObject>(obj: T): T { - const array = !Array.isArray(obj) ? [obj] : obj; - const camelCasedArray = array.map((item) => { - return mapKeys(item, (value, key) => camelCase(key)); - }); + if (typeof obj !== "object") { + return obj; + } - return (!Array.isArray(obj) ? camelCasedArray[0] : camelCasedArray) as T; + if (Array.isArray(obj)) { + return obj.map((item) => camelCaseObject(item)) as unknown as T; + } + + const mappedRootKeys = mapKeys(obj, (value, key) => camelCase(key)) as T; + + for (const key in mappedRootKeys) { + if (typeof mappedRootKeys[key] === "object") { + (mappedRootKeys[key] as unknown) = camelCaseObject(mappedRootKeys[key] as any); // eslint-disable-line @typescript-eslint/no-explicit-any + } + } + + return mappedRootKeys; } diff --git a/api/src/contribution/repository.ts b/api/src/contribution/repository.ts index a98e450bf..e2d171e93 100644 --- a/api/src/contribution/repository.ts +++ b/api/src/contribution/repository.ts @@ -2,6 +2,7 @@ import { ne, sql } from "drizzle-orm"; import { camelCaseObject } from "src/_utils/case"; import { reverseHierarchy } from "src/_utils/reverse-hierarchy"; import { unStringifyDeep } from "src/_utils/unstringify-deep"; +import { contributorsTable } from "src/contributor/table"; import { projectsTable } from "src/project/table"; import { repositoriesTable } from "src/repository/table"; import { SQLiteService } from "src/sqlite/service"; @@ -47,16 +48,44 @@ export class ContributionRepository { r.name as name, r.project_id as project_id, json_group_array( - json_object('id', c.id, 'title', c.title, 'type', c.type, 'url', c.url, 'updated_at', c.updated_at, 'activity_count', c.activity_count) + json_object( + 'id', + c.id, + 'title', + c.title, + 'type', + c.type, + 'url', + c.url, + 'updated_at', + c.updated_at, + 'activity_count', + c.activity_count, + 'contributor', + json_object( + 'id', + cr.id, + 'name', + cr.name, + 'username', + cr.username, + 'avatar_url', + cr.avatar_url + ) + ) ) AS contributions FROM ${contributionsTable} c - RIGHT JOIN - ${repositoriesTable} r ON c.id = r.project_id + INNER JOIN + ${repositoriesTable} r ON c.repository_id = r.id + INNER JOIN + ${contributorsTable} cr ON c.contributor_id = cr.id GROUP BY c.id) AS r - RIGHT JOIN + INNER JOIN ${projectsTable} p ON r.project_id = p.id + GROUP BY + p.id `; const raw = this.sqliteService.db.all(statement); @@ -69,6 +98,10 @@ export class ContributionRepository { const camelCased = camelCaseObject(reversed); - return camelCased; + const sortedUpdatedAt = camelCased.sort((a: any, b: any) => { + return new Date(b.updatedAt).getTime() - new Date(a.updatedAt).getTime(); + }); + + return sortedUpdatedAt; } } diff --git a/api/src/contribution/types.ts b/api/src/contribution/types.ts index c6562ccc8..1061f8dd6 100644 --- a/api/src/contribution/types.ts +++ b/api/src/contribution/types.ts @@ -1,4 +1,5 @@ import { ContributionEntity } from "@dzcode.io/models/dist/contribution"; +import { ContributorEntity } from "@dzcode.io/models/dist/contributor"; import { ProjectEntity } from "@dzcode.io/models/dist/project"; import { RepositoryEntity } from "@dzcode.io/models/dist/repository"; import { Transform, TransformFnParams, Type } from "class-transformer"; @@ -35,6 +36,7 @@ export interface GetContributionsResponseDto extends GeneralResponseDto { repository: Pick & { project: Pick; }; + contributor: Pick; } >; } diff --git a/api/src/digest/cron.ts b/api/src/digest/cron.ts index cda12c1c8..39aa75f40 100644 --- a/api/src/digest/cron.ts +++ b/api/src/digest/cron.ts @@ -61,6 +61,8 @@ export class DigestCron { * Generate a random runId, use it to tag all newly fetched data, persist it to the database, then delete all data that doesn't have that runId. */ private async run() { + if (Math.random() >= 0) return; + const runId = Math.random().toString(36).slice(2); this.logger.info({ message: `Digest cron started, runId: ${runId}` }); @@ -71,6 +73,7 @@ export class DigestCron { for (const project of projectsFromDataFolder) { const [{ id: projectId }] = await this.projectsRepository.upsert({ ...project, runId }); + let addedRepositoryCount = 0; try { const repositoriesFromDataFolder = project.repositories; for (const repository of repositoriesFromDataFolder) { @@ -87,6 +90,7 @@ export class DigestCron { runId, projectId, }); + addedRepositoryCount++; const issues = await this.githubService.listRepositoryIssues({ owner: repository.owner, @@ -126,6 +130,11 @@ export class DigestCron { // @TODO-ZM: capture error console.error(error); } + + if (addedRepositoryCount === 0) { + captureException(new Error("Empty project"), { extra: { project } }); + await this.projectsRepository.deleteById(projectId); + } } await this.contributorsRepository.deleteAllButWithRunId(runId); diff --git a/api/src/project/repository.ts b/api/src/project/repository.ts index 272ee40bd..7e0618502 100644 --- a/api/src/project/repository.ts +++ b/api/src/project/repository.ts @@ -1,5 +1,5 @@ import { ProjectEntity } from "@dzcode.io/models/dist/project"; -import { ne, sql } from "drizzle-orm"; +import { eq, ne, sql } from "drizzle-orm"; import { repositoriesTable } from "src/repository/table"; import { SQLiteService } from "src/sqlite/service"; import { Service } from "typedi"; @@ -51,6 +51,10 @@ export class ProjectRepository { .returning({ id: projectsTable.id }); } + public async deleteById(id: number) { + return await this.sqliteService.db.delete(projectsTable).where(eq(projectsTable.id, id)); + } + public async deleteAllButWithRunId(runId: string) { return await this.sqliteService.db.delete(projectsTable).where(ne(projectsTable.runId, runId)); } diff --git a/web/src/pages/contribute/index.tsx b/web/src/pages/contribute/index.tsx index ed7762943..abf8924aa 100644 --- a/web/src/pages/contribute/index.tsx +++ b/web/src/pages/contribute/index.tsx @@ -54,10 +54,9 @@ export default function Page(): JSX.Element {

- - {contribution.repository.project.name} - - + + {contribution.repository.project.name} + {contribution.repository.owner}/{contribution.repository.name} {/*
@@ -73,10 +72,10 @@ export default function Page(): JSX.Element { ))}
*/}
- {/* */} + src={contribution.contributor.avatarUrl} + />
{contribution.activityCount > 0 && (
From 79c2c842c770c03b11783fca653853a862f57196 Mon Sep 17 00:00:00 2001 From: Zakaria Mansouri Date: Sat, 7 Sep 2024 19:59:30 +0200 Subject: [PATCH 06/12] projects endpoint tweaks --- api/src/_utils/case.ts | 2 +- api/src/_utils/unstringify-deep.ts | 8 ++++---- api/src/digest/cron.ts | 2 -- api/src/project/repository.ts | 18 ++++++------------ api/src/project/types.ts | 7 ++++++- 5 files changed, 17 insertions(+), 20 deletions(-) diff --git a/api/src/_utils/case.ts b/api/src/_utils/case.ts index 709c38e00..212f42055 100644 --- a/api/src/_utils/case.ts +++ b/api/src/_utils/case.ts @@ -1,7 +1,7 @@ import camelCase from "lodash/camelCase"; import mapKeys from "lodash/mapKeys"; -export function camelCaseObject>(obj: T): T { +export function camelCaseObject(obj: T): T { if (typeof obj !== "object") { return obj; } diff --git a/api/src/_utils/unstringify-deep.ts b/api/src/_utils/unstringify-deep.ts index 61f5bbe36..8ac0490b0 100644 --- a/api/src/_utils/unstringify-deep.ts +++ b/api/src/_utils/unstringify-deep.ts @@ -3,10 +3,10 @@ * its children. */ -// eslint-disable-next-line @typescript-eslint/no-unnecessary-type-constraint -export function unStringifyDeep(obj: T): T { +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export function unStringifyDeep(obj: any): any { if (Array.isArray(obj)) { - return obj.map((item) => unStringifyDeep(item)) as T; + return obj.map((item) => unStringifyDeep(item)) as unknown as typeof obj; } if (typeof obj !== "object" || obj === null) { @@ -30,5 +30,5 @@ export function unStringifyDeep(obj: T): T { } } - return result as T; + return result as typeof obj; } diff --git a/api/src/digest/cron.ts b/api/src/digest/cron.ts index 39aa75f40..037999264 100644 --- a/api/src/digest/cron.ts +++ b/api/src/digest/cron.ts @@ -61,8 +61,6 @@ export class DigestCron { * Generate a random runId, use it to tag all newly fetched data, persist it to the database, then delete all data that doesn't have that runId. */ private async run() { - if (Math.random() >= 0) return; - const runId = Math.random().toString(36).slice(2); this.logger.info({ message: `Digest cron started, runId: ${runId}` }); diff --git a/api/src/project/repository.ts b/api/src/project/repository.ts index 7e0618502..3bff57ccd 100644 --- a/api/src/project/repository.ts +++ b/api/src/project/repository.ts @@ -1,5 +1,6 @@ -import { ProjectEntity } from "@dzcode.io/models/dist/project"; import { eq, ne, sql } from "drizzle-orm"; +import { camelCaseObject } from "src/_utils/case"; +import { unStringifyDeep } from "src/_utils/unstringify-deep"; import { repositoriesTable } from "src/repository/table"; import { SQLiteService } from "src/sqlite/service"; import { Service } from "typedi"; @@ -27,17 +28,10 @@ export class ProjectRepository { GROUP BY p.id; `; - const raw = this.sqliteService.db.all(statement) as Array< - // the SQL query above returns a stringified JSON for the `repositories` column - Omit & { repositories: string } - >; - const projectsForList: ProjectEntity[] = raw.map((row) => { - const notYetValid = { ...row, repositories: JSON.parse(row.repositories) }; - - return notYetValid; - }); - - return projectsForList; + const raw = this.sqliteService.db.all(statement); + const unStringifiedRaw = unStringifyDeep(raw); + const camelCased = camelCaseObject(unStringifiedRaw); + return camelCased; } public async upsert(project: ProjectRow) { diff --git a/api/src/project/types.ts b/api/src/project/types.ts index cf2cc84fa..ccd5d703f 100644 --- a/api/src/project/types.ts +++ b/api/src/project/types.ts @@ -1,6 +1,11 @@ import { ProjectEntity } from "@dzcode.io/models/dist/project"; +import { RepositoryEntity } from "@dzcode.io/models/dist/repository"; import { GeneralResponseDto } from "src/app/types"; export interface GetProjectsResponseDto extends GeneralResponseDto { - projects: ProjectEntity[]; + projects: Array< + Pick & { + repositories: Array>; + } + >; } From cd4598fe3c5dfa294dc9c82ad85156a543eb9959 Mon Sep 17 00:00:00 2001 From: Zakaria Mansouri Date: Sat, 7 Sep 2024 20:03:29 +0200 Subject: [PATCH 07/12] removed unused exports --- api/src/_utils/reverse-hierarchy.ts | 1 + api/src/contribution/repository.ts | 1 + api/src/contribution/types.ts | 51 ----------------------------- api/src/github/types.ts | 22 +------------ 4 files changed, 3 insertions(+), 72 deletions(-) diff --git a/api/src/_utils/reverse-hierarchy.ts b/api/src/_utils/reverse-hierarchy.ts index e1d08e0e8..eb9673e7d 100644 --- a/api/src/_utils/reverse-hierarchy.ts +++ b/api/src/_utils/reverse-hierarchy.ts @@ -4,6 +4,7 @@ export function reverseHierarchy( _obj: unknown, foreignKeyParentKeyRecord: ForeignKeyParentKeyRecord[], parentWithItsKey: Record = {}, + // eslint-disable-next-line @typescript-eslint/no-explicit-any ): any { if (Array.isArray(_obj)) { return _obj diff --git a/api/src/contribution/repository.ts b/api/src/contribution/repository.ts index e2d171e93..5010475c7 100644 --- a/api/src/contribution/repository.ts +++ b/api/src/contribution/repository.ts @@ -98,6 +98,7 @@ export class ContributionRepository { const camelCased = camelCaseObject(reversed); + // eslint-disable-next-line @typescript-eslint/no-explicit-any const sortedUpdatedAt = camelCased.sort((a: any, b: any) => { return new Date(b.updatedAt).getTime() - new Date(a.updatedAt).getTime(); }); diff --git a/api/src/contribution/types.ts b/api/src/contribution/types.ts index 1061f8dd6..bf80b2292 100644 --- a/api/src/contribution/types.ts +++ b/api/src/contribution/types.ts @@ -2,34 +2,8 @@ import { ContributionEntity } from "@dzcode.io/models/dist/contribution"; import { ContributorEntity } from "@dzcode.io/models/dist/contributor"; import { ProjectEntity } from "@dzcode.io/models/dist/project"; import { RepositoryEntity } from "@dzcode.io/models/dist/repository"; -import { Transform, TransformFnParams, Type } from "class-transformer"; -import { IsBoolean, IsIn, IsOptional, IsString, ValidateNested } from "class-validator"; import { GeneralResponseDto } from "src/app/types"; -export class OptionDto { - @IsString() - @IsOptional() - label?: string; - - @IsString() - name!: string; - - @IsBoolean() - @IsOptional() - checked?: boolean; -} - -export const allFilterNames = ["projects", "languages", "labels"] as const; - -export class FilterDto { - @IsIn(allFilterNames) - name!: (typeof allFilterNames)[number]; - - @ValidateNested({ each: true }) - @Type(() => OptionDto) - options!: OptionDto[]; -} - export interface GetContributionsResponseDto extends GeneralResponseDto { contributions: Array< Pick & { @@ -40,28 +14,3 @@ export interface GetContributionsResponseDto extends GeneralResponseDto { } >; } - -const transformFilterOptions = ({ value }: TransformFnParams) => { - let filterOptions: string[] = []; - if (typeof value === "string" && value.length > 0) { - filterOptions = value.split(","); - } - if (Array.isArray(value)) { - filterOptions = value; - } - return filterOptions; -}; - -export class GetContributionsQueryDto { - @Transform(transformFilterOptions) - @Reflect.metadata("design:type", { name: "string" }) - projects: string[] = []; - - @Transform(transformFilterOptions) - @Reflect.metadata("design:type", { name: "string" }) - languages: string[] = []; - - @Transform(transformFilterOptions) - @Reflect.metadata("design:type", { name: "string" }) - labels: string[] = []; -} diff --git a/api/src/github/types.ts b/api/src/github/types.ts index 05f197077..d3e5fc77c 100644 --- a/api/src/github/types.ts +++ b/api/src/github/types.ts @@ -85,31 +85,11 @@ export interface GetRepositoryInput { repo: string; } -export interface GitHubListRepositoryIssuesInput { +interface GitHubListRepositoryIssuesInput { owner: string; repository: string; } -export interface GithubIssue { - html_url: string; - number: number; - title: string; - user: GithubUser; - body: string; - labels: Array<{ - name: string; - }>; - state: "closed" | "open"; - assignees: GithubUser[]; - comments: number; - created_at: string; - updated_at: string; - closed_at: string | null; - pull_request?: { - html_url: string; - }; -} - export type GitHubListRepositoryLanguagesInput = GitHubListRepositoryIssuesInput; export type GitHubListRepositoryMilestonesInput = GitHubListRepositoryIssuesInput; From 1406be132d6e26314bc843f08af385f4f0cd69db Mon Sep 17 00:00:00 2001 From: Zakaria Mansouri Date: Sun, 8 Sep 2024 12:13:08 +0200 Subject: [PATCH 08/12] model and create rel between contor and contion --- api/db/migrations/0001_black_eternals.sql | 10 + api/db/migrations/meta/0001_snapshot.json | 386 ++++++++++++++++++++++ api/db/migrations/meta/_journal.json | 7 + api/src/contribution/repository.ts | 4 +- api/src/contribution/types.ts | 1 + api/src/contributor/repository.ts | 105 +++++- api/src/contributor/table.ts | 29 +- api/src/contributor/types.ts | 17 + api/src/digest/cron.ts | 21 +- 9 files changed, 570 insertions(+), 10 deletions(-) create mode 100644 api/db/migrations/0001_black_eternals.sql create mode 100644 api/db/migrations/meta/0001_snapshot.json create mode 100644 api/src/contributor/types.ts diff --git a/api/db/migrations/0001_black_eternals.sql b/api/db/migrations/0001_black_eternals.sql new file mode 100644 index 000000000..c3109ac28 --- /dev/null +++ b/api/db/migrations/0001_black_eternals.sql @@ -0,0 +1,10 @@ +CREATE TABLE `contributor_repository_relation` ( + `contributor_id` integer NOT NULL, + `repository_id` integer NOT NULL, + `record_imported_at` text DEFAULT CURRENT_TIMESTAMP NOT NULL, + `run_id` text DEFAULT 'initial-run-id' NOT NULL, + `score` integer NOT NULL, + PRIMARY KEY(`contributor_id`, `repository_id`), + FOREIGN KEY (`contributor_id`) REFERENCES `contributors`(`id`) ON UPDATE no action ON DELETE no action, + FOREIGN KEY (`repository_id`) REFERENCES `repositories`(`id`) ON UPDATE no action ON DELETE no action +); diff --git a/api/db/migrations/meta/0001_snapshot.json b/api/db/migrations/meta/0001_snapshot.json new file mode 100644 index 000000000..67f027972 --- /dev/null +++ b/api/db/migrations/meta/0001_snapshot.json @@ -0,0 +1,386 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "d99b3c61-9212-4dde-814c-d4cbac5ea54d", + "prevId": "ba41012f-4495-42ff-ace1-61bd7eaef476", + "tables": { + "contributions": { + "name": "contributions", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "record_imported_at": { + "name": "record_imported_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "activity_count": { + "name": "activity_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "repository_id": { + "name": "repository_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "contributor_id": { + "name": "contributor_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "contributions_url_unique": { + "name": "contributions_url_unique", + "columns": ["url"], + "isUnique": true + } + }, + "foreignKeys": { + "contributions_repository_id_repositories_id_fk": { + "name": "contributions_repository_id_repositories_id_fk", + "tableFrom": "contributions", + "tableTo": "repositories", + "columnsFrom": ["repository_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "contributions_contributor_id_contributors_id_fk": { + "name": "contributions_contributor_id_contributors_id_fk", + "tableFrom": "contributions", + "tableTo": "contributors", + "columnsFrom": ["contributor_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "contributor_repository_relation": { + "name": "contributor_repository_relation", + "columns": { + "contributor_id": { + "name": "contributor_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "repository_id": { + "name": "repository_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "record_imported_at": { + "name": "record_imported_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + }, + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'initial-run-id'" + }, + "score": { + "name": "score", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "contributor_repository_relation_contributor_id_contributors_id_fk": { + "name": "contributor_repository_relation_contributor_id_contributors_id_fk", + "tableFrom": "contributor_repository_relation", + "tableTo": "contributors", + "columnsFrom": ["contributor_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "contributor_repository_relation_repository_id_repositories_id_fk": { + "name": "contributor_repository_relation_repository_id_repositories_id_fk", + "tableFrom": "contributor_repository_relation", + "tableTo": "repositories", + "columnsFrom": ["repository_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "contributor_repository_relation_pk": { + "columns": ["contributor_id", "repository_id"], + "name": "contributor_repository_relation_pk" + } + }, + "uniqueConstraints": {} + }, + "contributors": { + "name": "contributors", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "record_imported_at": { + "name": "record_imported_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + }, + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'initial-run-id'" + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "username": { + "name": "username", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "avatar_url": { + "name": "avatar_url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "contributors_url_unique": { + "name": "contributors_url_unique", + "columns": ["url"], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "projects": { + "name": "projects", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "record_imported_at": { + "name": "record_imported_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'initial-run-id'" + } + }, + "indexes": { + "projects_slug_unique": { + "name": "projects_slug_unique", + "columns": ["slug"], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + }, + "repositories": { + "name": "repositories", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "record_imported_at": { + "name": "record_imported_at", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "owner": { + "name": "owner", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'initial-run-id'" + }, + "project_id": { + "name": "project_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "repositories_provider_owner_name_unique": { + "name": "repositories_provider_owner_name_unique", + "columns": ["provider", "owner", "name"], + "isUnique": true + } + }, + "foreignKeys": { + "repositories_project_id_projects_id_fk": { + "name": "repositories_project_id_projects_id_fk", + "tableFrom": "repositories", + "tableTo": "projects", + "columnsFrom": ["project_id"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {} + } + }, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} diff --git a/api/db/migrations/meta/_journal.json b/api/db/migrations/meta/_journal.json index 434d06b88..7da2d65ac 100644 --- a/api/db/migrations/meta/_journal.json +++ b/api/db/migrations/meta/_journal.json @@ -8,6 +8,13 @@ "when": 1725654548149, "tag": "0000_sudden_doctor_doom", "breakpoints": true + }, + { + "idx": 1, + "version": "6", + "when": 1725790243766, + "tag": "0001_black_eternals", + "breakpoints": true } ] } diff --git a/api/src/contribution/repository.ts b/api/src/contribution/repository.ts index 5010475c7..c0d7891c4 100644 --- a/api/src/contribution/repository.ts +++ b/api/src/contribution/repository.ts @@ -12,9 +12,7 @@ import { ContributionRow, contributionsTable } from "./table"; @Service() export class ContributionRepository { - constructor(private readonly sqliteService: SQLiteService) { - this.findForList(); - } + constructor(private readonly sqliteService: SQLiteService) {} public async upsert(contribution: ContributionRow) { return await this.sqliteService.db diff --git a/api/src/contribution/types.ts b/api/src/contribution/types.ts index bf80b2292..a2e3c5190 100644 --- a/api/src/contribution/types.ts +++ b/api/src/contribution/types.ts @@ -4,6 +4,7 @@ import { ProjectEntity } from "@dzcode.io/models/dist/project"; import { RepositoryEntity } from "@dzcode.io/models/dist/repository"; import { GeneralResponseDto } from "src/app/types"; +// @TODO-ZM: remove "dto" from all interfaces export interface GetContributionsResponseDto extends GeneralResponseDto { contributions: Array< Pick & { diff --git a/api/src/contributor/repository.ts b/api/src/contributor/repository.ts index 45ae3d382..adebca4c3 100644 --- a/api/src/contributor/repository.ts +++ b/api/src/contributor/repository.ts @@ -1,13 +1,89 @@ -import { ne } from "drizzle-orm"; +import { ne, sql } from "drizzle-orm"; +import { camelCaseObject } from "src/_utils/case"; +import { unStringifyDeep } from "src/_utils/unstringify-deep"; +import { projectsTable } from "src/project/table"; +import { repositoriesTable } from "src/repository/table"; import { SQLiteService } from "src/sqlite/service"; import { Service } from "typedi"; -import { ContributorRow, contributorsTable } from "./table"; +import { + ContributorRepositoryRelationRow, + contributorRepositoryRelationTable, + ContributorRow, + contributorsTable, +} from "./table"; @Service() export class ContributorRepository { constructor(private readonly sqliteService: SQLiteService) {} + public async findForList() { + const statement = sql` + SELECT + sum(c.score) as score, + cr.id as id, + cr.name as name, + cr.username as username, + cr.url as url, + cr.avatar_url as avatar_url, + json_group_array( + json_object( + 'id', + p.id, + 'name', + p.name, + 'repositories', + c.repositories + ) + ) AS projects + FROM + (SELECT + sum(crr.score) as score, + crr.contributor_id as contributor_id, + crr.project_id as project_id, + json_group_array( + json_object( + 'id', + r.id, + 'owner', + r.owner, + 'name', + r.name + ) + ) AS repositories + FROM + (SELECT + contributor_id, + repository_id, + score, + r.project_id as project_id + FROM + ${contributorRepositoryRelationTable} crr + INNER JOIN + ${repositoriesTable} r ON crr.repository_id = r.id + ) as crr + INNER JOIN + ${repositoriesTable} r ON crr.repository_id = r.id + GROUP BY + crr.contributor_id, crr.project_id) as c + INNER JOIN + ${contributorsTable} cr ON c.contributor_id = cr.id + INNER JOIN + ${projectsTable} p ON c.project_id = p.id + GROUP BY + c.contributor_id + ORDER BY + score DESC + `; + + const raw = this.sqliteService.db.all(statement); + const unStringifiedRaw = unStringifyDeep(raw); + + const camelCased = camelCaseObject(unStringifiedRaw); + + return camelCased; + } + public async upsert(contributor: ContributorRow) { return await this.sqliteService.db .insert(contributorsTable) @@ -19,6 +95,31 @@ export class ContributorRepository { .returning({ id: contributorsTable.id }); } + public async upsertRelationWithRepository( + contributorRelationWithRepository: ContributorRepositoryRelationRow, + ) { + return await this.sqliteService.db + .insert(contributorRepositoryRelationTable) + .values(contributorRelationWithRepository) + .onConflictDoUpdate({ + target: [ + contributorRepositoryRelationTable.contributorId, + contributorRepositoryRelationTable.repositoryId, + ], + set: contributorRelationWithRepository, + }) + .returning({ + contributorId: contributorRepositoryRelationTable.contributorId, + repositoryId: contributorRepositoryRelationTable.repositoryId, + }); + } + + public async deleteAllRelationWithRepositoryButWithRunId(runId: string) { + return await this.sqliteService.db + .delete(contributorRepositoryRelationTable) + .where(ne(contributorRepositoryRelationTable.runId, runId)); + } + public async deleteAllButWithRunId(runId: string) { return await this.sqliteService.db .delete(contributorsTable) diff --git a/api/src/contributor/table.ts b/api/src/contributor/table.ts index 72be1d5e1..571200fde 100644 --- a/api/src/contributor/table.ts +++ b/api/src/contributor/table.ts @@ -1,7 +1,8 @@ import { Model } from "@dzcode.io/models/dist/_base"; import { ContributorEntity } from "@dzcode.io/models/dist/contributor"; import { sql } from "drizzle-orm"; -import { integer, sqliteTable, text } from "drizzle-orm/sqlite-core"; +import { integer, primaryKey, sqliteTable, text } from "drizzle-orm/sqlite-core"; +import { repositoriesTable } from "src/repository/table"; export const contributorsTable = sqliteTable("contributors", { id: integer("id", { mode: "number" }).primaryKey({ autoIncrement: true }), @@ -18,3 +19,29 @@ export const contributorsTable = sqliteTable("contributors", { contributorsTable.$inferSelect satisfies Model; export type ContributorRow = typeof contributorsTable.$inferInsert; + +export const contributorRepositoryRelationTable = sqliteTable( + "contributor_repository_relation", + { + contributorId: integer("contributor_id") + .notNull() + .references(() => contributorsTable.id), + repositoryId: integer("repository_id") + .notNull() + .references(() => repositoriesTable.id), + recordImportedAt: text("record_imported_at") + .notNull() + .default(sql`CURRENT_TIMESTAMP`), + runId: text("run_id").notNull().default("initial-run-id"), + score: integer("score").notNull(), + }, + (table) => ({ + pk: primaryKey({ + name: "contributor_repository_relation_pk", + columns: [table.contributorId, table.repositoryId], + }), + }), +); + +export type ContributorRepositoryRelationRow = + typeof contributorRepositoryRelationTable.$inferInsert; diff --git a/api/src/contributor/types.ts b/api/src/contributor/types.ts new file mode 100644 index 000000000..1048b0f71 --- /dev/null +++ b/api/src/contributor/types.ts @@ -0,0 +1,17 @@ +import { ContributorEntity } from "@dzcode.io/models/dist/contributor"; +import { ProjectEntity } from "@dzcode.io/models/dist/project"; +import { RepositoryEntity } from "@dzcode.io/models/dist/repository"; +import { GeneralResponseDto } from "src/app/types"; + +export interface GetContributorsResponseDto extends GeneralResponseDto { + contributors: Array< + Pick & { + projects: Array< + Pick & { + repositories: Array>; + } + >; + score: number; + } + >; +} diff --git a/api/src/digest/cron.ts b/api/src/digest/cron.ts index 037999264..0849b2ee7 100644 --- a/api/src/digest/cron.ts +++ b/api/src/digest/cron.ts @@ -105,6 +105,13 @@ export class DigestCron { runId, }); + await this.contributorsRepository.upsertRelationWithRepository({ + contributorId, + repositoryId, + runId, + score: 1, + }); + const type = issue.pull_request ? "PULL_REQUEST" : "ISSUE"; const [{ id: contributionId }] = await this.contributionsRepository.upsert({ title: issue.title, @@ -135,10 +142,16 @@ export class DigestCron { } } - await this.contributorsRepository.deleteAllButWithRunId(runId); - await this.contributionsRepository.deleteAllButWithRunId(runId); - await this.repositoriesRepository.deleteAllButWithRunId(runId); - await this.projectsRepository.deleteAllButWithRunId(runId); + try { + await this.contributorsRepository.deleteAllRelationWithRepositoryButWithRunId(runId); + await this.contributorsRepository.deleteAllButWithRunId(runId); + await this.contributionsRepository.deleteAllButWithRunId(runId); + await this.repositoriesRepository.deleteAllButWithRunId(runId); + await this.projectsRepository.deleteAllButWithRunId(runId); + } catch (error) { + // @TODO-ZM: capture error + console.error(error); + } this.logger.info({ message: `Digest cron finished, runId: ${runId}` }); } From 5d88230b522e0b007975e595eae385b8754d5d75 Mon Sep 17 00:00:00 2001 From: Zakaria Mansouri Date: Sun, 8 Sep 2024 12:34:01 +0200 Subject: [PATCH 09/12] contributors endpoint --- api/src/app/endpoints.ts | 7 +++---- api/src/app/index.ts | 2 ++ api/src/contributor/controller.ts | 20 ++++++++++++++++++++ web/src/pages/team/index.tsx | 19 +++++++++++-------- web/src/redux/actions/contributions.ts | 2 +- web/src/redux/actions/contributors.ts | 2 +- web/src/redux/slices/contributors-page.ts | 4 ++-- 7 files changed, 40 insertions(+), 16 deletions(-) create mode 100644 api/src/contributor/controller.ts diff --git a/api/src/app/endpoints.ts b/api/src/app/endpoints.ts index 0cf5e8fa6..0ced0b68c 100644 --- a/api/src/app/endpoints.ts +++ b/api/src/app/endpoints.ts @@ -1,9 +1,9 @@ import { GetArticleResponseDto, GetArticlesResponseDto } from "src/article/types"; import { GetContributionsResponseDto } from "src/contribution/types"; +import { GetContributorsResponseDto } from "src/contributor/types"; import { GetADocumentationResponseDto, GetDocumentationResponseDto } from "src/documentation/types"; import { GetMilestonesResponseDto } from "src/milestone/types"; import { GetProjectsResponseDto } from "src/project/types"; -import { GetTeamResponseDto } from "src/team/types"; // ts-prune-ignore-next export interface Endpoints { @@ -26,10 +26,9 @@ export interface Endpoints { }; "api:Contributions": { response: GetContributionsResponseDto; - query: [string, string][]; }; - "api:Team": { - response: GetTeamResponseDto; + "api:Contributors": { + response: GetContributorsResponseDto; }; "api:MileStones/dzcode": { response: GetMilestonesResponseDto; diff --git a/api/src/app/index.ts b/api/src/app/index.ts index b3ae1f3d9..b829808cc 100644 --- a/api/src/app/index.ts +++ b/api/src/app/index.ts @@ -9,6 +9,7 @@ import { createExpressServer, RoutingControllersOptions, useContainer } from "ro import { ArticleController } from "src/article/controller"; import { ConfigService } from "src/config/service"; import { ContributionController } from "src/contribution/controller"; +import { ContributorController } from "src/contributor/controller"; import { DigestCron } from "src/digest/cron"; import { DocumentationController } from "src/documentation/controller"; import { GithubController } from "src/github/controller"; @@ -47,6 +48,7 @@ export const routingControllersOptions: RoutingControllersOptions = { ProjectController, ArticleController, DocumentationController, + ContributorController, ], middlewares: [ SecurityMiddleware, diff --git a/api/src/contributor/controller.ts b/api/src/contributor/controller.ts new file mode 100644 index 000000000..60427a964 --- /dev/null +++ b/api/src/contributor/controller.ts @@ -0,0 +1,20 @@ +import { Controller, Get } from "routing-controllers"; +import { Service } from "typedi"; + +import { ContributorRepository } from "./repository"; +import { GetContributorsResponseDto } from "./types"; + +@Service() +@Controller("/Contributors") +export class ContributorController { + constructor(private readonly contributorRepository: ContributorRepository) {} + + @Get("/") + public async getContributors(): Promise { + const contributors = await this.contributorRepository.findForList(); + + return { + contributors, + }; + } +} diff --git a/web/src/pages/team/index.tsx b/web/src/pages/team/index.tsx index 134a69bcf..cc962423b 100644 --- a/web/src/pages/team/index.tsx +++ b/web/src/pages/team/index.tsx @@ -51,15 +51,18 @@ export default function Page(): JSX.Element { className="rounded-full w-20 h-20" />

{contributor.name}

-
    - {contributor.repositories.map((repository, repositoryIndex) => ( -
  • - - {getRepositoryName(repository)} - -
    • +
    + {contributor.projects.map((project, projectIndex) => ( +
    + {project.name} + {project.repositories.map((repository, repositoryIndex) => ( + + {getRepositoryName(repository)} + + ))} +
    ))} -
+
))} diff --git a/web/src/redux/actions/contributions.ts b/web/src/redux/actions/contributions.ts index 642530343..cf9ae31f3 100644 --- a/web/src/redux/actions/contributions.ts +++ b/web/src/redux/actions/contributions.ts @@ -8,7 +8,7 @@ export const fetchContributionsListAction = (): ThunkAction => async (dispatch) => { try { dispatch(contributionsPageSlice.actions.set({ contributionsList: null })); - const { contributions } = await fetchV2("api:Contributions", { query: [] }); + const { contributions } = await fetchV2("api:Contributions", {}); dispatch(contributionsPageSlice.actions.set({ contributionsList: contributions })); } catch (error) { diff --git a/web/src/redux/actions/contributors.ts b/web/src/redux/actions/contributors.ts index 9224adac2..f3e050c53 100644 --- a/web/src/redux/actions/contributors.ts +++ b/web/src/redux/actions/contributors.ts @@ -8,7 +8,7 @@ export const fetchContributorsListAction = (): ThunkAction => async (dispatch) => { try { dispatch(contributorsPageSlice.actions.set({ contributorsList: null })); - const { contributors } = await fetchV2("api:Team", {}); + const { contributors } = await fetchV2("api:Contributors", {}); dispatch(contributorsPageSlice.actions.set({ contributorsList: contributors })); } catch (error) { dispatch(contributorsPageSlice.actions.set({ contributorsList: "ERROR" })); diff --git a/web/src/redux/slices/contributors-page.ts b/web/src/redux/slices/contributors-page.ts index 47150399a..554642480 100644 --- a/web/src/redux/slices/contributors-page.ts +++ b/web/src/redux/slices/contributors-page.ts @@ -1,11 +1,11 @@ -import { GetTeamResponseDto } from "@dzcode.io/api/dist/team/types"; +import { GetContributorsResponseDto } from "@dzcode.io/api/dist/contributor/types"; import { createSlice } from "@reduxjs/toolkit"; import { setReducerFactory } from "src/redux/utils"; import { Loadable } from "src/utils/loadable"; // ts-prune-ignore-next export interface ContributorsPageState { - contributorsList: Loadable; + contributorsList: Loadable; } const initialState: ContributorsPageState = { From f85dad98bbec97e397be1ab0dfdb373b258ff3f5 Mon Sep 17 00:00:00 2001 From: Zakaria Mansouri Date: Sun, 8 Sep 2024 13:31:30 +0200 Subject: [PATCH 10/12] list all contributors --- api/src/contributor/repository.ts | 13 ++++++-- api/src/github/service.ts | 11 +++---- api/src/github/types.ts | 55 ++----------------------------- web/src/pages/team/index.tsx | 2 +- 4 files changed, 20 insertions(+), 61 deletions(-) diff --git a/api/src/contributor/repository.ts b/api/src/contributor/repository.ts index adebca4c3..2c010633f 100644 --- a/api/src/contributor/repository.ts +++ b/api/src/contributor/repository.ts @@ -32,6 +32,8 @@ export class ContributorRepository { p.id, 'name', p.name, + 'score', + c.score, 'repositories', c.repositories ) @@ -48,7 +50,9 @@ export class ContributorRepository { 'owner', r.owner, 'name', - r.name + r.name, + 'score', + crr.score ) ) AS repositories FROM @@ -61,11 +65,16 @@ export class ContributorRepository { ${contributorRepositoryRelationTable} crr INNER JOIN ${repositoriesTable} r ON crr.repository_id = r.id + ORDER BY + crr.score DESC ) as crr INNER JOIN ${repositoriesTable} r ON crr.repository_id = r.id GROUP BY - crr.contributor_id, crr.project_id) as c + crr.contributor_id, crr.project_id + ORDER BY + crr.score DESC + ) as c INNER JOIN ${contributorsTable} cr ON c.contributor_id = cr.id INNER JOIN diff --git a/api/src/github/service.ts b/api/src/github/service.ts index 45b8fb8cb..fbaf173a2 100644 --- a/api/src/github/service.ts +++ b/api/src/github/service.ts @@ -45,6 +45,8 @@ export class GithubService { const contributors = commits // @TODO-ZM: dry to a user block-list // excluding github.com/web-flow user + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore .filter((item) => item.committer && item.committer.id !== 19864447) .map(({ committer }) => committer); return contributors; @@ -116,12 +118,7 @@ export class GithubService { // @TODO-ZM: validate responses using DTOs, for all fetchService methods if (!Array.isArray(contributors)) return []; - return ( - contributors - // @TODO-ZM: filter out bots - .filter(({ type }) => type === "User") - .sort((a, b) => b.contributions - a.contributions) - ); + return contributors; }; public getRateLimit = async (): Promise<{ limit: number; used: number; ratio: number }> => { @@ -152,6 +149,8 @@ export class GithubService { }; public githubUserToAccountEntity = ( + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore user: Pick, ): Model => ({ id: `github/${user.id}`, diff --git a/api/src/github/types.ts b/api/src/github/types.ts index d3e5fc77c..ba19eaea6 100644 --- a/api/src/github/types.ts +++ b/api/src/github/types.ts @@ -3,61 +3,12 @@ import { GeneralResponseDto } from "src/app/types"; export interface GithubUser { login: string; - id: number; - node_id: string; - avatar_url: string; - gravatar_id: string; - url: string; - html_url: string; - followers_url: string; - following_url: string; - gists_url: string; - starred_url: string; - subscriptions_url: string; - organizations_url: string; - repos_url: string; - events_url: string; - received_events_url: string; - type: string; - site_admin: boolean; name: string; - company: string; - blog: string; - location: string; - email: string; - hireable: boolean; - bio: string; - twitter_username: string; - public_repos: number; - public_gists: number; - followers: number; - following: number; - created_at: string; - updated_at: string; + html_url: string; + avatar_url: string; } -export interface GithubRepositoryContributor - extends Pick< - GithubUser, - | "login" - | "id" - | "node_id" - | "avatar_url" - | "gravatar_id" - | "url" - | "html_url" - | "followers_url" - | "following_url" - | "gists_url" - | "starred_url" - | "subscriptions_url" - | "organizations_url" - | "repos_url" - | "events_url" - | "received_events_url" - | "type" - | "site_admin" - > { +export interface GithubRepositoryContributor extends GithubUser { contributions: number; } diff --git a/web/src/pages/team/index.tsx b/web/src/pages/team/index.tsx index cc962423b..df339866d 100644 --- a/web/src/pages/team/index.tsx +++ b/web/src/pages/team/index.tsx @@ -51,7 +51,7 @@ export default function Page(): JSX.Element { className="rounded-full w-20 h-20" />

{contributor.name}

-
+
{contributor.projects.map((project, projectIndex) => (
{project.name} From 56a63e0ce09e2a7fb600ad1d996d11095dbf21f2 Mon Sep 17 00:00:00 2001 From: Zakaria Mansouri Date: Sun, 8 Sep 2024 13:53:25 +0200 Subject: [PATCH 11/12] list all contributors that are actual users --- api/src/_test/mocks.ts | 31 ++--------------------------- api/src/article/controller.ts | 5 +++++ api/src/digest/cron.ts | 29 +++++++++++++++++++++++++++ api/src/documentation/controller.ts | 4 ++++ api/src/github/types.ts | 1 + api/src/team/repository.ts | 4 ++++ 6 files changed, 45 insertions(+), 29 deletions(-) diff --git a/api/src/_test/mocks.ts b/api/src/_test/mocks.ts index 34c50b2a5..a96d5fb81 100644 --- a/api/src/_test/mocks.ts +++ b/api/src/_test/mocks.ts @@ -3,35 +3,8 @@ import { GithubUser } from "src/github/types"; export const githubUserMock: GithubUser = { login: "ZibanPirate", - id: 20110076, - node_id: "MDQ6VXNlcjIwMTEwMDc2", - avatar_url: "https://avatars.githubusercontent.com/u/20110076?v=4", - gravatar_id: "", - url: "https://api.github.com/users/ZibanPirate", html_url: "https://github.com/ZibanPirate", - followers_url: "https://api.github.com/users/ZibanPirate/followers", - following_url: "https://api.github.com/users/ZibanPirate/following{/other_user}", - gists_url: "https://api.github.com/users/ZibanPirate/gists{/gist_id}", - starred_url: "https://api.github.com/users/ZibanPirate/starred{/owner}{/repo}", - subscriptions_url: "https://api.github.com/users/ZibanPirate/subscriptions", - organizations_url: "https://api.github.com/users/ZibanPirate/orgs", - repos_url: "https://api.github.com/users/ZibanPirate/repos", - events_url: "https://api.github.com/users/ZibanPirate/events{/privacy}", - received_events_url: "https://api.github.com/users/ZibanPirate/received_events", - type: "User", - site_admin: false, + avatar_url: "https://avatars.githubusercontent.com/u/20110076?v=4", name: "Zakaria Mansouri", - company: "@dzcode-io @avimedical", - blog: "zak.dzcode.io", - location: "Algeria", - email: "", - hireable: true, - bio: "One-man-army lone programmer", - twitter_username: "ZibanPirate", - public_repos: 18, - public_gists: 2, - followers: 130, - following: 92, - created_at: "2016-06-23T12:41:14Z", - updated_at: "2023-04-10T21:31:26Z", + type: "User", }; diff --git a/api/src/article/controller.ts b/api/src/article/controller.ts index 295f269fe..77b94bcb0 100644 --- a/api/src/article/controller.ts +++ b/api/src/article/controller.ts @@ -7,6 +7,7 @@ import { Service } from "typedi"; import { GetArticleResponseDto, GetArticlesResponseDto } from "./types"; +// @TODO-ZM: remove article and learn controllers @Service() @Controller("/Articles") export class ArticleController { @@ -42,6 +43,8 @@ export class ArticleController { const authors = await Promise.all( article.authors.map(async (author) => { const githubUser = await this.githubService.getUser({ username: author }); + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore return this.githubService.githubUserToAccountEntity(githubUser); }), ); @@ -77,6 +80,8 @@ export class ArticleController { } }, []) .sort((a, b) => uniqUsernames[b.login] - uniqUsernames[a.login]) + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore .map((committer) => this.githubService.githubUserToAccountEntity(committer)) .filter(({ id }) => !authors.find((author) => author.id === id)); diff --git a/api/src/digest/cron.ts b/api/src/digest/cron.ts index 0849b2ee7..754bda438 100644 --- a/api/src/digest/cron.ts +++ b/api/src/digest/cron.ts @@ -97,6 +97,9 @@ export class DigestCron { for (const issue of issues.issues) { const githubUser = await this.githubService.getUser({ username: issue.user.login }); + + if (githubUser.type !== "User") continue; + const [{ id: contributorId }] = await this.contributorsRepository.upsert({ name: githubUser.name || githubUser.login, username: githubUser.login, @@ -126,6 +129,32 @@ export class DigestCron { console.log("contributionId", contributionId); } + + const repoContributors = await this.githubService.listRepositoryContributors({ + owner: repository.owner, + repository: repository.name, + }); + + const repoContributorsFiltered = repoContributors.filter( + (contributor) => contributor.type === "User", + ); + + for (const repoContributor of repoContributorsFiltered) { + const [{ id: contributorId }] = await this.contributorsRepository.upsert({ + name: repoContributor.name || repoContributor.login, + username: repoContributor.login, + url: repoContributor.html_url, + avatarUrl: repoContributor.avatar_url, + runId, + }); + + await this.contributorsRepository.upsertRelationWithRepository({ + contributorId, + repositoryId, + runId, + score: repoContributor.contributions, + }); + } } catch (error) { // @TODO-ZM: capture error console.error(error); diff --git a/api/src/documentation/controller.ts b/api/src/documentation/controller.ts index 06fcf3708..51f823396 100644 --- a/api/src/documentation/controller.ts +++ b/api/src/documentation/controller.ts @@ -44,6 +44,8 @@ export class DocumentationController { const authors = await Promise.all( documentation.authors.map(async (author) => { const githubUser = await this.githubService.getUser({ username: author }); + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore return this.githubService.githubUserToAccountEntity(githubUser); }), ); @@ -79,6 +81,8 @@ export class DocumentationController { } }, []) .sort((a, b) => uniqUsernames[b.login] - uniqUsernames[a.login]) + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore .map((contributor) => this.githubService.githubUserToAccountEntity(contributor)) .filter(({ id }) => !authors.find((author) => author.id === id)); diff --git a/api/src/github/types.ts b/api/src/github/types.ts index ba19eaea6..be2a28076 100644 --- a/api/src/github/types.ts +++ b/api/src/github/types.ts @@ -6,6 +6,7 @@ export interface GithubUser { name: string; html_url: string; avatar_url: string; + type: "User" | "_other"; } export interface GithubRepositoryContributor extends GithubUser { diff --git a/api/src/team/repository.ts b/api/src/team/repository.ts index c72388575..df250abf0 100644 --- a/api/src/team/repository.ts +++ b/api/src/team/repository.ts @@ -43,6 +43,8 @@ export class TeamRepository { repository: name, }); contributors.forEach((contributor) => { + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore const uuid = this.githubService.githubUserToAccountEntity({ ...contributor, name: "", @@ -89,6 +91,8 @@ export class TeamRepository { .map(async (uuid) => { const { repositories, login } = contributorsUsernameRankedRecord[uuid]; const githubUser = await this.githubService.getUser({ username: login }); + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore const account = this.githubService.githubUserToAccountEntity(githubUser); return { ...account, repositories }; From 3f7d8e5ba6f0d8de2bba8c5d063686199f72e50d Mon Sep 17 00:00:00 2001 From: Zakaria Mansouri Date: Sun, 8 Sep 2024 14:59:54 +0200 Subject: [PATCH 12/12] updated snapshot tests --- .../project-reference/__snapshots__/index.spec.ts.snap | 8 ++++---- .../repository-reference/__snapshots__/index.spec.ts.snap | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/models/src/project-reference/__snapshots__/index.spec.ts.snap b/packages/models/src/project-reference/__snapshots__/index.spec.ts.snap index 42c998511..7f3643bbd 100644 --- a/packages/models/src/project-reference/__snapshots__/index.spec.ts.snap +++ b/packages/models/src/project-reference/__snapshots__/index.spec.ts.snap @@ -7,14 +7,14 @@ ProjectReferenceEntity { "name": "Leblad", "repositories": [ RepositoryReferenceEntity { + "name": "leblad", "owner": "dzcode-io", "provider": "github", - "repository": "leblad", }, RepositoryReferenceEntity { + "name": "leblad-py", "owner": "abderrahmaneMustapha", "provider": "github", - "repository": "leblad-py", }, ], "slug": "Leblad", @@ -28,14 +28,14 @@ ProjectReferenceEntity { "name": "Leblad", "repositories": [ RepositoryReferenceEntity { + "name": "leblad", "owner": "dzcode-io", "provider": "github", - "repository": "leblad", }, RepositoryReferenceEntity { + "name": "leblad-py", "owner": "abderrahmaneMustapha", "provider": "github", - "repository": "leblad-py", }, ], "slug": "Leblad", diff --git a/packages/models/src/repository-reference/__snapshots__/index.spec.ts.snap b/packages/models/src/repository-reference/__snapshots__/index.spec.ts.snap index c9192cf93..15c39ca2f 100644 --- a/packages/models/src/repository-reference/__snapshots__/index.spec.ts.snap +++ b/packages/models/src/repository-reference/__snapshots__/index.spec.ts.snap @@ -6,9 +6,9 @@ exports[`should match snapshot when providing all fields: output 1`] = ` RepositoryReferenceEntity { "contributions": [], "contributors": [], + "name": "leblad", "owner": "dzcode-io", "provider": "github", - "repository": "leblad", } `; @@ -16,9 +16,9 @@ exports[`should match snapshot when providing required fields only: errors 1`] = exports[`should match snapshot when providing required fields only: output 1`] = ` RepositoryReferenceEntity { + "name": "leblad", "owner": "dzcode-io", "provider": "github", - "repository": "leblad", } `; @@ -45,9 +45,9 @@ exports[`should show an error that matches snapshot when passing empty object: e ValidationError { "children": [], "constraints": { - "isString": "repository must be a string", + "isString": "name must be a string", }, - "property": "repository", + "property": "name", "target": RepositoryReferenceEntity {}, "value": undefined, },