diff --git a/frontend/src/app/assignment/assignment.module.ts b/frontend/src/app/assignment/assignment.module.ts index 006c76ed3..7d8ca94a6 100644 --- a/frontend/src/app/assignment/assignment.module.ts +++ b/frontend/src/app/assignment/assignment.module.ts @@ -22,7 +22,6 @@ import {TokenInterceptor} from "./services/token.interceptor"; import {AssignmentService} from "./services/assignment.service"; import {TokenService} from "./services/token.service"; import {SolutionService} from "./services/solution.service"; -import {TelemetryService} from "./services/telemetry.service"; import {CourseService} from "./services/course.service"; import {SelectionService} from "./services/selection.service"; import {SolutionContainerService} from "./services/solution-container.service"; @@ -70,7 +69,6 @@ import {KeycloakBearerInterceptor} from "keycloak-angular"; ConfigService, AssignmentService, SolutionService, - TelemetryService, CourseService, SelectionService, SolutionContainerService, diff --git a/frontend/src/app/assignment/model/evaluation.ts b/frontend/src/app/assignment/model/evaluation.ts index 088c53515..790024196 100644 --- a/frontend/src/app/assignment/model/evaluation.ts +++ b/frontend/src/app/assignment/model/evaluation.ts @@ -28,6 +28,7 @@ export class Evaluation { author: string; remark: string; points: number; + duration?: number; snippets: Snippet[]; codeSearch?: CodeSearchInfo; diff --git a/frontend/src/app/assignment/model/telemetry.ts b/frontend/src/app/assignment/model/telemetry.ts deleted file mode 100644 index eddb37f3e..000000000 --- a/frontend/src/app/assignment/model/telemetry.ts +++ /dev/null @@ -1,15 +0,0 @@ -export interface Telemetry { - assignment: string; - solution?: string; - evaluation?: string; - task?: string; - - timestamp: Date; - - createdBy?: string; - author?: string; - - action: string; -} - -export type CreateTelemetryDto = Omit; diff --git a/frontend/src/app/assignment/modules/assignment/solution-table/solution-table.component.html b/frontend/src/app/assignment/modules/assignment/solution-table/solution-table.component.html index f42189578..3ab77556a 100644 --- a/frontend/src/app/assignment/modules/assignment/solution-table/solution-table.component.html +++ b/frontend/src/app/assignment/modules/assignment/solution-table/solution-table.component.html @@ -161,7 +161,6 @@ routerLink="/assignments/{{assignment?._id}}/solutions/{{solution._id}}" queryParamsHandling="merge" ngbTooltip="View Solution" - (click)="telemetry(solution, 'openSolution')" >
diff --git a/frontend/src/app/assignment/modules/assignment/solution-table/solution-table.component.ts b/frontend/src/app/assignment/modules/assignment/solution-table/solution-table.component.ts index 96fbc6d12..f601f436a 100644 --- a/frontend/src/app/assignment/modules/assignment/solution-table/solution-table.component.ts +++ b/frontend/src/app/assignment/modules/assignment/solution-table/solution-table.component.ts @@ -10,7 +10,6 @@ import Solution, {AuthorInfo, authorInfoProperties} from '../../../model/solutio import {AssignmentService} from '../../../services/assignment.service'; import {SolutionService} from '../../../services/solution.service'; import {TaskService} from '../../../services/task.service'; -import {TelemetryService} from '../../../services/telemetry.service'; import {SubmitService} from "../submit.service"; import {UserService} from "../../../../user/user.service"; import {AssigneeService} from "../../../services/assignee.service"; @@ -56,7 +55,6 @@ export class SolutionTableComponent implements OnInit { private assigneeService: AssigneeService, private evaluationService: EvaluationService, private router: Router, - private telemetryService: TelemetryService, private activatedRoute: ActivatedRoute, private toastService: ToastService, private taskService: TaskService, @@ -192,13 +190,6 @@ export class SolutionTableComponent implements OnInit { return [...valueSet].sort(); } - telemetry(solution: Solution, action: string, timestamp = new Date()) { - this.telemetryService.create(solution.assignment, solution._id!, { - action, - timestamp, - }).subscribe(); - } - copyPoints() { this.clipboardService.copy(this.solutions!.map(s => s.points ?? '').join('\n')); this.toastService.success('Copy Points', `Copied ${this.solutions.length} rows to clipboard`); @@ -235,7 +226,6 @@ export class SolutionTableComponent implements OnInit { return; } - const timestamp = new Date(); const result = await Promise.all(this.solutions .filter(s => this.selected[s._id!] && s.author.github) .map(async solution => { @@ -243,8 +233,6 @@ export class SolutionTableComponent implements OnInit { await this.submitService.postIssueToGitHub(assignment, solution, issue, userToken); solution.points = issue._points; - this.telemetry(solution, 'submitFeedback', timestamp); - return solution; }) ); diff --git a/frontend/src/app/assignment/modules/assignment/statistics/statistics.component.html b/frontend/src/app/assignment/modules/assignment/statistics/statistics.component.html index 9bd74fa2e..35ec54a23 100644 --- a/frontend/src/app/assignment/modules/assignment/statistics/statistics.component.html +++ b/frontend/src/app/assignment/modules/assignment/statistics/statistics.component.html @@ -78,10 +78,10 @@

- +
@@ -97,7 +97,7 @@

popoverTitle="Code Search Time Savings Explained" > - +
@@ -167,7 +167,7 @@

  • The task took - {{ taskStats.timeAvg / 1000 | duration }} + {{ taskStats.timeAvg | duration }} on average to evaluate.
  • @@ -213,7 +213,7 @@

  • Code Search saved approximately - {{ taskStats._codeSearchTimeSavings / 1000 | duration }} + {{ taskStats._codeSearchTimeSavings | duration }} of grading time for this task.
    • diff --git a/frontend/src/app/assignment/modules/assignment/statistics/statistics.component.ts b/frontend/src/app/assignment/modules/assignment/statistics/statistics.component.ts index 32fdfa532..ce512f3d8 100644 --- a/frontend/src/app/assignment/modules/assignment/statistics/statistics.component.ts +++ b/frontend/src/app/assignment/modules/assignment/statistics/statistics.component.ts @@ -37,7 +37,7 @@ export class StatisticsComponent implements OnInit { title: 'Average Evaluation Time', label: 'per Evaluation', get: t => t.timeAvg, - render: n => this.durationPipe.transform(n / 1000), + render: n => this.durationPipe.transform(n), }, codeSearchEffectiveness: { title: 'Code Search Effectiveness', @@ -49,7 +49,7 @@ export class StatisticsComponent implements OnInit { title: 'Code Search Time Savings', label: 'Saved by Code Search', get: t => t._codeSearchTimeSavings, - render: n => this.durationPipe.transform(n / 1000), + render: n => this.durationPipe.transform(n), }, } as const; visibleProps = new Set(['score', 'codeSearchTimeSavings']); diff --git a/frontend/src/app/assignment/modules/assignment/submit-modal/submit-modal.component.ts b/frontend/src/app/assignment/modules/assignment/submit-modal/submit-modal.component.ts index f597e60ba..ca7c6c4ec 100644 --- a/frontend/src/app/assignment/modules/assignment/submit-modal/submit-modal.component.ts +++ b/frontend/src/app/assignment/modules/assignment/submit-modal/submit-modal.component.ts @@ -8,7 +8,6 @@ import {ReadAssignmentDto} from '../../../model/assignment'; import Solution from '../../../model/solution'; import {AssignmentService} from '../../../services/assignment.service'; import {SolutionService} from '../../../services/solution.service'; -import {TelemetryService} from '../../../services/telemetry.service'; import {IssueDto, SubmitService} from '../submit.service'; @Component({ @@ -30,7 +29,6 @@ export class SubmitModalComponent implements OnInit { constructor( public route: ActivatedRoute, private assignmentService: AssignmentService, - private telemetryService: TelemetryService, private solutionService: SolutionService, private submitService: SubmitService, private toastService: ToastService, @@ -73,10 +71,6 @@ export class SubmitModalComponent implements OnInit { } const {assignment, _id} = this.solution!; - this.telemetryService.create(assignment, _id!, { - action: 'submitFeedback', - timestamp: new Date(), - }).subscribe(); this.submitting = true; this.solutionService.update(assignment, _id!, { diff --git a/frontend/src/app/assignment/modules/shared/task-list/task-list.component.html b/frontend/src/app/assignment/modules/shared/task-list/task-list.component.html index aeaf70aae..437f51332 100644 --- a/frontend/src/app/assignment/modules/shared/task-list/task-list.component.html +++ b/frontend/src/app/assignment/modules/shared/task-list/task-list.component.html @@ -28,7 +28,6 @@ [class.bi-robot]="evaluation.author === 'Code Search'" [ngbTooltip]="evaluation.remark + ' ~ ' + evaluation.author" [routerLink]="[task._id]" - (click)="openTelemetry(task)" > ; constructor( - private telemetryService: TelemetryService, private evaluationService: EvaluationService, private configService: ConfigService, private toastService: ToastService, @@ -29,15 +26,6 @@ export class TaskListComponent { ) { } - openTelemetry(task: Task) { - const {aid, sid} = this.route.snapshot.params; - this.telemetryService.create(aid, sid, { - task: task._id, - timestamp: new Date(), - action: 'openEvaluation', - }).subscribe(); - } - givePoints(task: Task, points: number) { const {aid, sid} = this.route.snapshot.params; this.evaluationService.findByTask(aid, sid, task._id).pipe( diff --git a/frontend/src/app/assignment/modules/solution/edit-snippet/edit-snippet.component.ts b/frontend/src/app/assignment/modules/solution/edit-snippet/edit-snippet.component.ts index 584beb191..af3c56b31 100644 --- a/frontend/src/app/assignment/modules/solution/edit-snippet/edit-snippet.component.ts +++ b/frontend/src/app/assignment/modules/solution/edit-snippet/edit-snippet.component.ts @@ -2,7 +2,7 @@ import {Component, EventEmitter, Input, Output} from '@angular/core'; import {merge, OperatorFunction, Subject} from 'rxjs'; import {debounceTime, distinctUntilChanged, map} from 'rxjs/operators'; import {Snippet} from '../../../model/evaluation'; -import {selectionComment} from '../evaluation-modal/evaluation-modal.component'; +import {selectionComment} from '../snippet-list/snippet-list.component'; @Component({ selector: 'app-edit-snippet', diff --git a/frontend/src/app/assignment/modules/solution/evaluation-modal/evaluation-modal.component.html b/frontend/src/app/assignment/modules/solution/evaluation-modal/evaluation-modal.component.html index feceb0192..33719fb77 100644 --- a/frontend/src/app/assignment/modules/solution/evaluation-modal/evaluation-modal.component.html +++ b/frontend/src/app/assignment/modules/solution/evaluation-modal/evaluation-modal.component.html @@ -17,63 +17,14 @@ . - -
    Code Search derived {{ derivedSolutionCount }} evaluations from this. View in Table
    -
    - Code Search found - {{ searchSummary.hits }} - hits in - {{ searchSummary.files }} - files from - {{ searchSummary.solutions }} - solutions for this snippet. - {{ searchSummary.message }} - - View Results - -
    -
      -
    • - - -
      • -
    -
    - Use - - fulibFeedback - - to add code snippets from within your IDE. - - Code Search will only look for files matching the glob pattern {{ task.glob }}. - -
    - - -
      -
    • - -
      • -
    -     + +
    diff --git a/frontend/src/app/assignment/modules/solution/evaluation-modal/evaluation-modal.component.ts b/frontend/src/app/assignment/modules/solution/evaluation-modal/evaluation-modal.component.ts index cd5a6236e..7ca7590b7 100644 --- a/frontend/src/app/assignment/modules/solution/evaluation-modal/evaluation-modal.component.ts +++ b/frontend/src/app/assignment/modules/solution/evaluation-modal/evaluation-modal.component.ts @@ -1,22 +1,19 @@ import {Component, HostListener, OnDestroy, OnInit, ViewChild} from '@angular/core'; import {ActivatedRoute, Router} from '@angular/router'; import {ModalComponent, ToastService} from '@mean-stream/ngbx'; -import {EMPTY, merge, of, Subject, Subscription} from 'rxjs'; -import {debounceTime, distinctUntilChanged, filter, map, share, switchMap, tap} from 'rxjs/operators'; -import {CodeSearchInfo, CreateEvaluationDto, Evaluation, Snippet} from '../../../model/evaluation'; -import {SearchSummary} from '../../../model/search-result'; +import {EMPTY, Observable, of, Subscription} from 'rxjs'; +import {map, share, switchMap, tap} from 'rxjs/operators'; +import {CodeSearchInfo, CreateEvaluationDto, Evaluation} from '../../../model/evaluation'; import Solution from '../../../model/solution'; import Task from '../../../model/task'; import {AssignmentService} from '../../../services/assignment.service'; import {ConfigService} from '../../../services/config.service'; import {SolutionService} from '../../../services/solution.service'; import {TaskService} from '../../../services/task.service'; -import {TelemetryService} from '../../../services/telemetry.service'; -import {SelectionService} from '../../../services/selection.service'; import {EvaluationService} from "../../../services/evaluation.service"; -import {EmbeddingService} from "../../../services/embedding.service"; +import {selectionComment} from "../snippet-list/snippet-list.component"; +import {ReadAssignmentDto} from "../../../model/assignment"; -export const selectionComment = '(fulibFeedback Selection)'; @Component({ selector: 'app-evaluation-modal', @@ -26,14 +23,11 @@ export const selectionComment = '(fulibFeedback Selection)'; export class EvaluationModalComponent implements OnInit, OnDestroy { @ViewChild('modal', {static: true}) modal: ModalComponent; - readonly selectionComment = selectionComment; - codeSearchEnabled = this.configService.getBool('codeSearch'); - snippetSuggestionsEnabled = this.configService.getBool('snippetSuggestions'); similarSolutionsEnabled = this.configService.getBool('similarSolutions'); + startDate = Date.now(); task?: Task; - comments: string[] = []; evaluation?: Evaluation; dto: CreateEvaluationDto = { task: '', @@ -48,11 +42,6 @@ export class EvaluationModalComponent implements OnInit, OnDestroy { derivedSolutionCount?: number; - snippetUpdates$ = new Subject(); - searchSummary?: SearchSummary & { level: string, message?: string, code: string }; - - embeddingSnippets: Snippet[] = []; - viewSimilar = true; subscriptions = new Subscription(); @@ -61,32 +50,46 @@ export class EvaluationModalComponent implements OnInit, OnDestroy { private assignmentService: AssignmentService, private taskService: TaskService, private solutionService: SolutionService, - private selectionService: SelectionService, private configService: ConfigService, private toastService: ToastService, - private telemetryService: TelemetryService, private evaluationService: EvaluationService, - private embeddingService: EmbeddingService, public route: ActivatedRoute, private router: Router, ) { } ngOnInit(): void { - this.route.params.pipe( - switchMap(({aid, task}) => this.assignmentService.get(aid).pipe( - tap(assignment => this.dto.codeSearch = this.codeSearchEnabled && !!assignment.classroom?.codeSearch), - map(assignment => this.taskService.find(assignment.tasks, task)), - )), - ).subscribe(task => { - this.task = task; - }); + const assignment$ = this.route.params.pipe( + switchMap(({aid}) => this.assignmentService.get(aid)), + share(), + ); + + this.loadCodeSearchEnabled(assignment$); + this.loadTask(assignment$); const evaluation$ = this.route.params.pipe( switchMap(({aid, sid, task}) => this.evaluationService.findByTask(aid, sid, task)), share(), ); + this.loadEvaluation(evaluation$); + this.loadOriginEvaluationAndSolution(evaluation$); + this.loadDerivedSolutions(evaluation$); + } + + private loadCodeSearchEnabled(assignment$: Observable) { + this.subscriptions.add(assignment$.subscribe(assignment => { + this.dto.codeSearch = this.codeSearchEnabled && !!assignment.classroom?.codeSearch; + })); + } + + private loadTask(assignment$: Observable) { + this.subscriptions.add(assignment$.subscribe(assignment => { + this.task = this.taskService.find(assignment.tasks, this.route.snapshot.params.task); + })); + } + + private loadEvaluation(evaluation$: Observable) { this.subscriptions.add(evaluation$.subscribe(evaluation => { this.evaluation = evaluation; if (evaluation) { @@ -94,7 +97,9 @@ export class EvaluationModalComponent implements OnInit, OnDestroy { this.dto = {...this.dto, points, remark, snippets}; } })); + } + private loadOriginEvaluationAndSolution(evaluation$: Observable) { this.subscriptions.add(evaluation$.pipe( switchMap(evaluation => { const origin = evaluation?.codeSearch?.origin; @@ -104,75 +109,15 @@ export class EvaluationModalComponent implements OnInit, OnDestroy { switchMap(originEvaluation => originEvaluation ? this.solutionService.get(originEvaluation.assignment, originEvaluation.solution) : of(undefined)), tap(originSolution => this.originSolution = originSolution), ).subscribe()); + } + private loadDerivedSolutions(evaluation$: Observable) { this.subscriptions.add(evaluation$.pipe( switchMap(evaluation => evaluation ? this.evaluationService.distinctValues(evaluation.assignment, 'solution', { origin: evaluation._id, task: evaluation.task, }) : EMPTY), ).subscribe(solutionIds => this.derivedSolutionCount = solutionIds.length)); - - this.route.params.pipe( - switchMap(({aid, task}) => this.evaluationService.distinctValues(aid, 'snippets.comment', {task})), - ).subscribe(comments => this.comments = comments); - - if (this.snippetSuggestionsEnabled) { - this.route.params.pipe( - switchMap(({aid, sid, task}) => this.embeddingService.findTaskRelatedSnippets(aid, sid, task)), - ).subscribe(snippets => this.embeddingSnippets = snippets); - } - - const selection$ = this.route.params.pipe( - switchMap(({aid, sid}) => this.selectionService.stream(aid, sid)), - filter(({selection: {author}}) => author === this.dto.author), - map(({selection}) => selection), - filter(({snippet}) => !!snippet.code.trim()), - share(), - ); - - this.subscriptions.add(selection$.subscribe(({author, snippet}) => { - let index = this.dto.snippets.findIndex(s => s.comment === this.selectionComment); - if (index >= 0) { - this.dto.snippets[index] = snippet; - } else { - index = this.dto.snippets.push(snippet) - 1; - } - setTimeout(() => document.getElementById('snippet-' + index)?.focus()); - })); - - if (this.codeSearchEnabled) { - this.subscriptions.add(merge( - selection$.pipe(map(sel => sel.snippet.code)), - this.snippetUpdates$.pipe(map(snippet => snippet.pattern || snippet.code)), - ).pipe( - debounceTime(200), - distinctUntilChanged(), - switchMap(code => this.assignmentService.searchSummary(this.route.snapshot.params.aid, code, this.task?.glob, '***').pipe( - map(searchSummary => ({...searchSummary, code})), - )), - ).subscribe(summary => { - let level: string; - let message: string | undefined; - if (!summary.hits) { - level = 'warning'; - message = 'No result indicates the snippet is not part of the submitted code for this solution. Please make sure you checked out the correct commit.'; - } else if (summary.files > summary.solutions) { - level = 'danger'; - message = 'The snippet was found in multiple files per solution. It most likely does not provide enough context.'; - } else if (summary.hits > summary.files) { - level = 'warning'; - message = 'The snippet was found in multiple places per file. It probably does not provide enough context.'; - } else { - level = 'success'; - } - - this.searchSummary = { - ...summary, - level, - message, - }; - })); - } } ngOnDestroy(): void { @@ -187,29 +132,15 @@ export class EvaluationModalComponent implements OnInit, OnDestroy { } } - confirmEmbedding(snippet: Snippet) { - this.embeddingSnippets.splice(this.embeddingSnippets.indexOf(snippet), 1); - this.dto.snippets.push(snippet); - snippet.score = undefined; - this.snippetUpdates$.next(snippet); - } - - deleteSnippet(index: number) { - this.dto.snippets.splice(index, 1); - } - doSubmit(): void { const {aid, sid, task} = this.route.snapshot.params; this.dto.task = task; - this.dto.snippets.removeFirst(s => s.comment === this.selectionComment); + this.dto.snippets.removeFirst(s => s.comment === selectionComment); - this.telemetryService.create(aid, sid, { - timestamp: new Date(), - task, - author: this.dto.author, - action: 'submitEvaluation', - }).subscribe(); + if (!this.evaluation) { + this.dto.duration = (Date.now() - this.startDate) / 1000; + } const op = this.evaluation ? this.evaluationService.update(aid, sid, this.evaluation._id, this.dto) diff --git a/frontend/src/app/assignment/modules/solution/feedback/feedback.component.ts b/frontend/src/app/assignment/modules/solution/feedback/feedback.component.ts index ecb619f3a..180b2eaca 100644 --- a/frontend/src/app/assignment/modules/solution/feedback/feedback.component.ts +++ b/frontend/src/app/assignment/modules/solution/feedback/feedback.component.ts @@ -1,4 +1,4 @@ -import { Component } from '@angular/core'; +import {Component, OnInit} from '@angular/core'; import Solution, {Feedback} from "../../../model/solution"; import {ActivatedRoute} from "@angular/router"; import {SolutionService} from "../../../services/solution.service"; @@ -10,7 +10,7 @@ import {ToastService} from "@mean-stream/ngbx"; templateUrl: './feedback.component.html', styleUrls: ['./feedback.component.scss'] }) -export class FeedbackComponent { +export class FeedbackComponent implements OnInit { protected readonly Feedback = Feedback; feedback: Feedback = {}; diff --git a/frontend/src/app/assignment/modules/solution/snippet-list/snippet-list.component.html b/frontend/src/app/assignment/modules/solution/snippet-list/snippet-list.component.html new file mode 100644 index 000000000..5628aa874 --- /dev/null +++ b/frontend/src/app/assignment/modules/solution/snippet-list/snippet-list.component.html @@ -0,0 +1,50 @@ + +
    + Code Search found + {{ searchSummary.hits }} + hits in + {{ searchSummary.files }} + files from + {{ searchSummary.solutions }} + solutions for this snippet. + {{ searchSummary.message }} + + View Results + +
    +
      +
    • + + +
      • +
    +
    + Use + + fulibFeedback + + to add code snippets from within your IDE. + + Code Search will only look for files matching the glob pattern {{ task.glob }}. + +
    + + +
      +
    • + +
      • +
    +     diff --git a/frontend/src/app/assignment/modules/solution/snippet-list/snippet-list.component.scss b/frontend/src/app/assignment/modules/solution/snippet-list/snippet-list.component.scss new file mode 100644 index 000000000..e69de29bb diff --git a/frontend/src/app/assignment/modules/solution/snippet-list/snippet-list.component.ts b/frontend/src/app/assignment/modules/solution/snippet-list/snippet-list.component.ts new file mode 100644 index 000000000..d93021f7d --- /dev/null +++ b/frontend/src/app/assignment/modules/solution/snippet-list/snippet-list.component.ts @@ -0,0 +1,147 @@ +import {Component, Input, OnDestroy, OnInit} from '@angular/core'; +import {CreateEvaluationDto, Snippet} from "../../../model/evaluation"; +import {merge, Observable, Subject, Subscription} from "rxjs"; +import {SearchSummary} from "../../../model/search-result"; +import {debounceTime, distinctUntilChanged, filter, map, share, switchMap} from "rxjs/operators"; +import {AssignmentService} from "../../../services/assignment.service"; +import {SelectionDto, SelectionService} from "../../../services/selection.service"; +import {ConfigService} from "../../../services/config.service"; +import {EvaluationService} from "../../../services/evaluation.service"; +import {EmbeddingService} from "../../../services/embedding.service"; +import {ActivatedRoute} from "@angular/router"; +import Task from "../../../model/task"; + +export const selectionComment = '(fulibFeedback Selection)'; + +@Component({ + selector: 'app-snippet-list', + templateUrl: './snippet-list.component.html', + styleUrls: ['./snippet-list.component.scss'] +}) +export class SnippetListComponent implements OnInit, OnDestroy { + @Input({required: true}) task?: Task; + @Input({required: true}) dto: CreateEvaluationDto; + + comments: string[] = []; + + snippetUpdates$ = new Subject(); + searchSummary?: SearchSummary & { level: string, message?: string, code: string }; + + embeddingSnippets: Snippet[] = []; + + codeSearchEnabled = this.configService.getBool('codeSearch'); + snippetSuggestionsEnabled = this.configService.getBool('snippetSuggestions'); + + private subscriptions = new Subscription(); + + readonly selectionComment = selectionComment; + + constructor( + private assignmentService: AssignmentService, + private selectionService: SelectionService, + private configService: ConfigService, + private evaluationService: EvaluationService, + private embeddingService: EmbeddingService, + public route: ActivatedRoute, + ) { + } + + ngOnInit() { + this.loadCommentValues(); + + if (this.snippetSuggestionsEnabled) { + this.loadEmbeddingRelatedSnippets(); + } + + const selection$ = this.route.params.pipe( + switchMap(({aid, sid}) => this.selectionService.stream(aid, sid)), + filter(({selection: {author}}) => author === this.dto.author), + map(({selection}) => selection), + filter(({snippet}) => !!snippet.code.trim()), + share(), + ); + + this.listenForSelectionSnippets(selection$); + + if (this.codeSearchEnabled) { + this.listenForCodeSearch(selection$); + } + } + + private loadCommentValues() { + this.route.params.pipe( + switchMap(({aid, task}) => this.evaluationService.distinctValues(aid, 'snippets.comment', {task})), + ).subscribe(comments => this.comments = comments); + } + + private loadEmbeddingRelatedSnippets() { + this.route.params.pipe( + switchMap(({aid, sid, task}) => this.embeddingService.findTaskRelatedSnippets(aid, sid, task)), + ).subscribe(snippets => this.embeddingSnippets = snippets); + } + + private listenForSelectionSnippets(selection$: Observable) { + this.subscriptions.add(selection$.subscribe(({snippet}) => { + let index = this.dto.snippets.findIndex(s => s.comment === this.selectionComment); + if (index >= 0) { + this.dto.snippets[index] = snippet; + } else { + index = this.dto.snippets.push(snippet) - 1; + } + setTimeout(() => document.getElementById('snippet-' + index)?.focus()); + })); + } + + private listenForCodeSearch(selection$: Observable) { + this.subscriptions.add(merge( + selection$.pipe(map(sel => sel.snippet.code)), + this.snippetUpdates$.pipe(map(snippet => snippet.pattern || snippet.code)), + ).pipe( + debounceTime(200), + distinctUntilChanged(), + switchMap(code => this.assignmentService.searchSummary(this.route.snapshot.params.aid, code, this.task?.glob, '***').pipe( + map(searchSummary => ({...searchSummary, code})), + )), + ).subscribe(summary => { + this.setSummary(summary); + })); + } + + private setSummary(summary: SearchSummary & { code: string }) { + let level: string; + let message: string | undefined; + if (!summary.hits) { + level = 'warning'; + message = 'No result indicates the snippet is not part of the submitted code for this solution. Please make sure you checked out the correct commit.'; + } else if (summary.files > summary.solutions) { + level = 'danger'; + message = 'The snippet was found in multiple files per solution. It most likely does not provide enough context.'; + } else if (summary.hits > summary.files) { + level = 'warning'; + message = 'The snippet was found in multiple places per file. It probably does not provide enough context.'; + } else { + level = 'success'; + } + + this.searchSummary = { + ...summary, + level, + message, + }; + } + + ngOnDestroy(): void { + this.subscriptions.unsubscribe(); + } + + confirmEmbedding(snippet: Snippet) { + this.embeddingSnippets.splice(this.embeddingSnippets.indexOf(snippet), 1); + this.dto.snippets.push(snippet); + snippet.score = undefined; + this.snippetUpdates$.next(snippet); + } + + deleteSnippet(index: number) { + this.dto.snippets.splice(index, 1); + } +} diff --git a/frontend/src/app/assignment/modules/solution/solution.module.ts b/frontend/src/app/assignment/modules/solution/solution.module.ts index dfba0f25f..986ec9864 100644 --- a/frontend/src/app/assignment/modules/solution/solution.module.ts +++ b/frontend/src/app/assignment/modules/solution/solution.module.ts @@ -22,6 +22,7 @@ import {CommentService} from "./comment.service"; import {FeedbackComponent} from './feedback/feedback.component'; import {TimetrackingComponent} from './timetracking/timetracking.component'; import {AssigneeFeedbackComponent} from './assignee-feedback/assignee-feedback.component'; +import { SnippetListComponent } from './snippet-list/snippet-list.component'; @NgModule({ declarations: [ @@ -38,6 +39,7 @@ import {AssigneeFeedbackComponent} from './assignee-feedback/assignee-feedback.c FeedbackComponent, TimetrackingComponent, AssigneeFeedbackComponent, + SnippetListComponent, ], imports: [ CommonModule, diff --git a/frontend/src/app/assignment/services/selection.service.ts b/frontend/src/app/assignment/services/selection.service.ts index 93abfad72..1840f5722 100644 --- a/frontend/src/app/assignment/services/selection.service.ts +++ b/frontend/src/app/assignment/services/selection.service.ts @@ -5,7 +5,7 @@ import {Snippet} from '../model/evaluation'; import {AssignmentService} from './assignment.service'; import {observeSSE} from './sse-helper'; -interface SelectionDto { +export interface SelectionDto { assignment: string; solution: string; author: string; diff --git a/frontend/src/app/assignment/services/telemetry.service.ts b/frontend/src/app/assignment/services/telemetry.service.ts deleted file mode 100644 index 5b98c95a1..000000000 --- a/frontend/src/app/assignment/services/telemetry.service.ts +++ /dev/null @@ -1,17 +0,0 @@ -import {HttpClient} from '@angular/common/http'; -import {Injectable} from '@angular/core'; -import {Observable} from 'rxjs'; -import {environment} from '../../../environments/environment'; -import {CreateTelemetryDto, Telemetry} from '../model/telemetry'; - -@Injectable() -export class TelemetryService { - constructor( - private http: HttpClient, - ) { - } - - create(assignment: string, solution: string, dto: CreateTelemetryDto): Observable { - return this.http.post(`${environment.assignmentsApiUrl}/assignments/${assignment}/solutions/${solution}/telemetry`, dto); - } -} diff --git a/services/apps/assignments/src/assignments.module.ts b/services/apps/assignments/src/assignments.module.ts index d224a09a4..7d804e255 100644 --- a/services/apps/assignments/src/assignments.module.ts +++ b/services/apps/assignments/src/assignments.module.ts @@ -14,7 +14,6 @@ import {SearchModule} from './search/search.module'; import {SelectionModule} from './selection/selection.module'; import {SolutionModule} from './solution/solution.module'; import {StatisticsModule} from './statistics/statistics.module'; -import {TelemetryModule} from './telemetry/telemetry.module'; import {SentryInterceptor, SentryModule} from "@ntegral/nestjs-sentry"; import {APP_INTERCEPTOR} from "@nestjs/core"; import {EmbeddingModule} from './embedding/embedding.module'; @@ -49,7 +48,6 @@ import {MemberModule} from "./member/member.module"; SearchModule, StatisticsModule, SelectionModule, - TelemetryModule, MossModule, EmbeddingModule, FileModule, diff --git a/services/apps/assignments/src/course/course.service.ts b/services/apps/assignments/src/course/course.service.ts index e6c847a46..4c43fdd79 100644 --- a/services/apps/assignments/src/course/course.service.ts +++ b/services/apps/assignments/src/course/course.service.ts @@ -2,7 +2,7 @@ import {EventService} from '@mean-stream/nestx'; import {Injectable} from '@nestjs/common'; import {InjectModel} from '@nestjs/mongoose'; import {FilterQuery, Model} from 'mongoose'; -import {AuthorInfo, Solution} from '../solution/solution.schema'; +import {AuthorInfo} from '../solution/solution.schema'; import {SolutionService} from '../solution/solution.service'; import {idFilter} from '../utils'; import {CourseStudent, CreateCourseDto, UpdateCourseDto} from './course.dto'; diff --git a/services/apps/assignments/src/evaluation/evaluation.schema.ts b/services/apps/assignments/src/evaluation/evaluation.schema.ts index b297271c5..b5177a66b 100644 --- a/services/apps/assignments/src/evaluation/evaluation.schema.ts +++ b/services/apps/assignments/src/evaluation/evaluation.schema.ts @@ -8,6 +8,7 @@ import { IsNotEmpty, IsNumber, IsOptional, + IsPositive, IsString, IsUUID, Min, @@ -136,6 +137,12 @@ export class Evaluation { @IsNumber() points: number; + @Prop() + @ApiPropertyOptional() + @IsOptional() + @IsPositive() + duration?: number; + @Prop() @ApiProperty({type: [Snippet]}) @ValidateNested({each: true}) diff --git a/services/apps/assignments/src/statistics/statistics.module.ts b/services/apps/assignments/src/statistics/statistics.module.ts index 69c60ea01..5b0563e5c 100644 --- a/services/apps/assignments/src/statistics/statistics.module.ts +++ b/services/apps/assignments/src/statistics/statistics.module.ts @@ -3,7 +3,6 @@ import {AssignmentModule} from '../assignment/assignment.module'; import {CommentModule} from '../comment/comment.module'; import {EvaluationModule} from '../evaluation/evaluation.module'; import {SolutionModule} from '../solution/solution.module'; -import {TelemetryModule} from '../telemetry/telemetry.module'; import {StatisticsController} from './statistics.controller'; import {StatisticsService} from './statistics.service'; @@ -13,7 +12,6 @@ import {StatisticsService} from './statistics.service'; EvaluationModule, SolutionModule, CommentModule, - TelemetryModule, ], controllers: [StatisticsController], providers: [StatisticsService], diff --git a/services/apps/assignments/src/statistics/statistics.service.ts b/services/apps/assignments/src/statistics/statistics.service.ts index 2300f7462..21c6a062c 100644 --- a/services/apps/assignments/src/statistics/statistics.service.ts +++ b/services/apps/assignments/src/statistics/statistics.service.ts @@ -4,10 +4,9 @@ import {AssignmentService} from '../assignment/assignment.service'; import {CommentService} from '../comment/comment.service'; import {EvaluationService} from '../evaluation/evaluation.service'; import {SolutionService} from '../solution/solution.service'; -import {TelemetryService} from '../telemetry/telemetry.service'; import {AssignmentStatistics, EvaluationStatistics, SolutionStatistics, TaskStatistics} from './statistics.dto'; -const outlierDurationMillis = 60 * 1000; +const outlierDuration = 60; @Injectable() export class StatisticsService { @@ -16,7 +15,6 @@ export class StatisticsService { private solutionService: SolutionService, private evaluationService: EvaluationService, private commentService: CommentService, - private telemetryService: TelemetryService, ) { } @@ -85,45 +83,20 @@ export class StatisticsService { let totalTime = 0; let weightedTime = 0; let codeSearchSavings = 0; - for (const result of await this.telemetryService.model.aggregate([ - {$match: {assignment, action: {$in: ['openEvaluation', 'submitEvaluation']}}}, - {$sort: {timestamp: 1}}, - {$group: {_id: {s: '$solution', t: '$task'} as any, events: {$push: '$$ROOT'}}}, + for (const result of await this.evaluationService.model.aggregate([ { - // end = events.filter(e => e.action === 'submitEvaluation') - $addFields: { - end: { - $last: { - $filter: { - input: '$events', cond: { - $eq: ['$$this.action', 'submitEvaluation'], - }, - }, - }, - }, + $match: { + assignment, + duration: {$lt: outlierDuration}, }, }, { - // start = events.filter(e => e.action === 'openEvaluation' && e.timestamp < end.timestamp) - $addFields: { - start: { - $last: { - $filter: { - input: '$events', - cond: { - $and: [ - {$eq: ['$$this.action', 'openEvaluation']}, - {$lt: ['$$this.timestamp', '$end.timestamp']}, - ], - }, - }, - }, - }, + $group: { + _id: '$task', + time: {$sum: '$duration'}, + count: {$sum: 1}, }, - }, - {$project: {duration: {$subtract: ['$end.timestamp', '$start.timestamp']}}}, - {$match: {duration: {$lt: outlierDurationMillis}}}, - {$group: {_id: '$_id.t' as any, time: {$sum: '$duration'}, count: {$sum: 1}}}, + } ])) { const {_id, time, count} = result; const taskStat = taskStats.get(_id); diff --git a/services/apps/assignments/src/telemetry/telemetry.controller.ts b/services/apps/assignments/src/telemetry/telemetry.controller.ts deleted file mode 100644 index 74c4eb334..000000000 --- a/services/apps/assignments/src/telemetry/telemetry.controller.ts +++ /dev/null @@ -1,30 +0,0 @@ -import {AuthUser, UserToken} from '@app/keycloak-auth'; -import {Body, Controller, Param, Post} from '@nestjs/common'; -import {ApiCreatedResponse, ApiTags} from '@nestjs/swagger'; -import {AssignmentAuth} from '../assignment/assignment-auth.decorator'; -import {CreateTelemetryDto} from './telemetry.dto'; -import {Telemetry} from './telemetry.schema'; -import {TelemetryService} from './telemetry.service'; - -const forbiddenResponse = 'Not owner of assignment, or invalid Assignment-Token.'; - -@Controller() -@ApiTags('Telemetry') -export class TelemetryController { - constructor( - private readonly telemetryService: TelemetryService, - ) { - } - - @Post('assignments/:assignment/solutions/:solution/telemetry') - @AssignmentAuth({forbiddenResponse}) - @ApiCreatedResponse({type: Telemetry}) - async create( - @Param('assignment') assignment: string, - @Param('solution') solution: string, - @Body() dto: CreateTelemetryDto, - @AuthUser() user?: UserToken, - ): Promise { - return this.telemetryService.create(assignment, solution, dto, user?.sub); - } -} diff --git a/services/apps/assignments/src/telemetry/telemetry.dto.ts b/services/apps/assignments/src/telemetry/telemetry.dto.ts deleted file mode 100644 index 99e7e0801..000000000 --- a/services/apps/assignments/src/telemetry/telemetry.dto.ts +++ /dev/null @@ -1,9 +0,0 @@ -import {OmitType} from '@nestjs/swagger'; -import {Telemetry} from './telemetry.schema'; - -export class CreateTelemetryDto extends OmitType(Telemetry, [ - 'assignment', - 'solution', - 'createdBy', -] as const) { -} diff --git a/services/apps/assignments/src/telemetry/telemetry.module.ts b/services/apps/assignments/src/telemetry/telemetry.module.ts deleted file mode 100644 index 9c7c91a10..000000000 --- a/services/apps/assignments/src/telemetry/telemetry.module.ts +++ /dev/null @@ -1,27 +0,0 @@ -import {Module} from '@nestjs/common'; -import {MongooseModule} from '@nestjs/mongoose'; -import {AssignmentModule} from '../assignment/assignment.module'; -import {SolutionModule} from '../solution/solution.module'; -import {TelemetryController} from './telemetry.controller'; -import {Telemetry, TelemetrySchema} from './telemetry.schema'; -import {TelemetryService} from './telemetry.service'; - -@Module({ - imports: [ - MongooseModule.forFeature([{ - name: Telemetry.name, - schema: TelemetrySchema, - }]), - AssignmentModule, - SolutionModule, - ], - controllers: [TelemetryController], - providers: [ - TelemetryService, - ], - exports: [ - TelemetryService, - ], -}) -export class TelemetryModule { -} diff --git a/services/apps/assignments/src/telemetry/telemetry.schema.ts b/services/apps/assignments/src/telemetry/telemetry.schema.ts deleted file mode 100644 index 692e0bbce..000000000 --- a/services/apps/assignments/src/telemetry/telemetry.schema.ts +++ /dev/null @@ -1,60 +0,0 @@ -import {Prop, Schema, SchemaFactory} from '@nestjs/mongoose'; -import {ApiProperty, ApiPropertyOptional} from '@nestjs/swagger'; -import {Transform} from 'class-transformer'; -import {IsAlphanumeric, IsDate, IsMongoId, IsNotEmpty, IsOptional, IsString} from 'class-validator'; -import {Document} from 'mongoose'; - -@Schema() -export class Telemetry { - @Prop() - @ApiProperty() - @IsMongoId() - assignment: string; - - @Prop() - @ApiPropertyOptional() - @IsOptional() - @IsMongoId() - solution?: string; - - @Prop() - @ApiPropertyOptional() - @IsOptional() - @IsAlphanumeric() - task?: string; - - @Prop() - @ApiPropertyOptional() - @IsOptional() - @IsMongoId() - evaluation?: string; - - @Prop() - @ApiProperty() - @IsDate() - @Transform(({value}) => typeof value === 'string' ? new Date(value) : value) - timestamp: Date; - - @Prop() - @ApiPropertyOptional() - @IsOptional() - @IsString() - @IsNotEmpty() - createdBy?: string; - - @Prop() - @ApiPropertyOptional() - @IsOptional() - @IsString() - author?: string; - - @Prop() - @ApiProperty() - @IsString() - @IsNotEmpty() - action: string; -} - -export type TelemetryDocument = Telemetry & Document; - -export const TelemetrySchema = SchemaFactory.createForClass(Telemetry); diff --git a/services/apps/assignments/src/telemetry/telemetry.service.ts b/services/apps/assignments/src/telemetry/telemetry.service.ts deleted file mode 100644 index d1a75d3d5..000000000 --- a/services/apps/assignments/src/telemetry/telemetry.service.ts +++ /dev/null @@ -1,34 +0,0 @@ -import {EventService} from '@mean-stream/nestx'; -import {Injectable} from '@nestjs/common'; -import {InjectModel} from '@nestjs/mongoose'; -import {FilterQuery, Model} from 'mongoose'; -import {CreateTelemetryDto} from './telemetry.dto'; -import {Telemetry, TelemetryDocument} from './telemetry.schema'; - -@Injectable() -export class TelemetryService { - constructor( - @InjectModel(Telemetry.name) public model: Model, - private eventService: EventService, - ) { - } - - private emit(event: 'created', telemetry: TelemetryDocument) { - this.eventService.emit(`.solutions.${telemetry.solution}.solutions.${telemetry.solution}.telemetry.${telemetry.id}.${event}`, telemetry); - } - - async create(assignment: string, solution: string, dto: CreateTelemetryDto, createdBy?: string): Promise { - const telemetry = await this.model.create({ - assignment, - solution, - createdBy, - ...dto, - }); - this.emit('created', telemetry); - return telemetry; - } - - async findAll(where: FilterQuery = {}): Promise { - return this.model.find(where).exec(); - } -} diff --git a/services/libs/member/src/member.schema.ts b/services/libs/member/src/member.schema.ts index f78d9d6e9..bbe33dce3 100644 --- a/services/libs/member/src/member.schema.ts +++ b/services/libs/member/src/member.schema.ts @@ -2,7 +2,6 @@ import {Ref} from '@mean-stream/nestx'; import {Prop, Schema, SchemaFactory} from '@nestjs/mongoose'; import {ApiProperty} from '@nestjs/swagger'; import {Document, Types} from 'mongoose'; -import {Project} from '../../../apps/projects/src/project/project.schema'; @Schema({_id: false, id: false}) export class Member {