From 7075d413dcd66f090fe5d609ba6d1e647ab3c04a Mon Sep 17 00:00:00 2001 From: Nathanael Shim Date: Thu, 29 Feb 2024 09:22:57 -0800 Subject: [PATCH 1/7] implement submission filter parity --- .../Submissions/Models/FilterConverter.cs | 19 +- .../Submissions/Models/ISubmission.cs | 49 +- .../Submissions/Models/Submission.cs | 174 +- .../Submissions/Models/SubmissionFilter.cs | 54 +- .../Submissions/Submissions.graphql | 61 +- IndicoV2.StrawberryShake/schema.graphql | 6614 +++++++++++++---- 6 files changed, 5407 insertions(+), 1564 deletions(-) diff --git a/IndicoV2.Abstractions/Submissions/Models/FilterConverter.cs b/IndicoV2.Abstractions/Submissions/Models/FilterConverter.cs index 9703d1ba..76618b4c 100644 --- a/IndicoV2.Abstractions/Submissions/Models/FilterConverter.cs +++ b/IndicoV2.Abstractions/Submissions/Models/FilterConverter.cs @@ -5,6 +5,7 @@ using v2SubmissionFilter = IndicoV2.Submissions.Models.SubmissionFilter; using ssSubmissionFilter = IndicoV2.StrawberryShake.SubmissionFilter; using System.Linq; +using IndicoV2.StrawberryShake; namespace IndicoV2.Submissions.Models { @@ -17,9 +18,25 @@ public static ssSubmissionFilter ConvertToSs(this IFilter filter) { return new ssSubmissionFilter() { + Filetype = submissionFilter.FileType.Select(x => (StrawberryShake.FileType)x).ToList(), InputFilename = submissionFilter.InputFilename, Retrieved = submissionFilter.Retrieved, - Status = (StrawberryShake.SubmissionStatus?)submissionFilter.Status + Status = (StrawberryShake.SubmissionStatus?)submissionFilter.Status, + Reviews = new ReviewFilter() { + Rejected = submissionFilter.Reviews.Rejected, + CreatedBy = submissionFilter.Reviews.CreatedBy, + ReviewType = (StrawberryShake.ReviewType) submissionFilter.Reviews.ReviewType + }, + ReviewInProgress = submissionFilter.ReviewInProgress, + FilesDeleted = submissionFilter.FilesDeleted, + CreatedAt = new StrawberryShake.DateRangeFilter() { + From = submissionFilter.CreatedAt.From, + To = submissionFilter.CreatedAt.To, + }, + UpdatedAt = new StrawberryShake.DateRangeFilter() { + From = submissionFilter.UpdatedAt.From, + To = submissionFilter.UpdatedAt.To + } }; } else if (filter is AndFilter andfilter) diff --git a/IndicoV2.Abstractions/Submissions/Models/ISubmission.cs b/IndicoV2.Abstractions/Submissions/Models/ISubmission.cs index fec56da3..a4a4beab 100644 --- a/IndicoV2.Abstractions/Submissions/Models/ISubmission.cs +++ b/IndicoV2.Abstractions/Submissions/Models/ISubmission.cs @@ -1,4 +1,5 @@  +using System; using IndicoV2.CommonModels.Pagination; namespace IndicoV2.Submissions.Models @@ -13,11 +14,6 @@ public interface ISubmission /// int Id { get; } - /// - /// Submission status. See . - /// - SubmissionStatus Status { get; } - /// /// Dataset id. /// @@ -28,6 +24,31 @@ public interface ISubmission /// int WorkflowId { get; } + /// + /// Submission status. See . + /// + SubmissionStatus Status { get; } + + DateTimeOffset? CreatedAt { get; } + + DateTimeOffset? UpdatedAt { get; } + + int? CreatedBy { get; } + + int? UpdatedBy { get; } + + DateTimeOffset? CompletedAt { get; } + + + /// + /// Submission errors. + /// + string Errors { get; } + + bool? FilesDeleted { get; } + + SubmissionFile[] InputFiles { get; } + /// /// Submission input file. /// @@ -43,14 +64,24 @@ public interface ISubmission /// string ResultFile { get; } + SubmissionOutput[] OutputFiles { get; } + /// /// Is submission retrieved. /// bool Retrieved { get; } - /// - /// Submission errors. - /// - string Errors { get; } + Review AutoReview { get; } + + SubmissionRetry[] Retries { get; } + + bool? AutoReviewLoaded { get; } + + string OcrEngine { get; } + + Review[] Reviews { get; } + + bool? ReviewInProgress { get; } + } } diff --git a/IndicoV2.Abstractions/Submissions/Models/Submission.cs b/IndicoV2.Abstractions/Submissions/Models/Submission.cs index a13ae8fb..9d457bf6 100644 --- a/IndicoV2.Abstractions/Submissions/Models/Submission.cs +++ b/IndicoV2.Abstractions/Submissions/Models/Submission.cs @@ -1,10 +1,75 @@ using System; -using System.Collections.Generic; -using System.Text; +using System.Linq; using IndicoV2.StrawberryShake; namespace IndicoV2.Submissions.Models { + public class SubmissionFile + { + public int? Id { get; set; } + + public string FilePath { get; set; } + + public string FileName { get; set; } + + public FileType? FileType { get; set; } + + public int? SubmissionId { get; set; } + + public int? FileSize { get; set; } + + public int? NumPages { get; set; } + } + + public class SubmissionOutput + { + public int? Id { get; set; } + + public string FilePath { get; set; } + + public int? SubmissionId { get; set; } + + public int? ComponentId { get; set; } + + public DateTimeOffset? CreatedAt { get; set; } + } + + public class Review + { + public int? Id { get; set; } + + public int? SubmissionId { get; set; } + + public string CreatedAt { get; set; } + + public int? CreatedBy { get; set; } + + public string StartedAt { get; set; } + + public string CompletedAt { get; set; } + + public bool? Rejected { get; set; } + + public ReviewType? ReviewType { get; set; } + + public string Notes { get; set; } + + public string Changes { get; set; } + } + + public class SubmissionRetry + { + public int? Id { get; set; } + + public int? SubmissionId { get; set; } + + public string PreviousErrors { get; set; } + + public SubmissionStatus? PreviousStatus { get; set; } + + public string RetryErrors { get; set; } + } + public class Submission : ISubmission { /// @@ -12,20 +77,42 @@ public class Submission : ISubmission /// public int Id { get; set; } + /// + /// Dataset id. + /// + public int DatasetId { get; set; } + + /// + /// Workflow id. + /// + public int WorkflowId { get; set; } + /// /// Submission status. See . /// public SubmissionStatus Status { get; set; } + public DateTimeOffset? CreatedAt { get; set; } + + public DateTimeOffset? UpdatedAt { get; set; } + + public int? CreatedBy { get; set; } + + public int? UpdatedBy { get; set; } + + public DateTimeOffset? CompletedAt { get; set; } + /// - /// Dataset id. + /// Submission errors. /// - public int DatasetId { get; set; } + public string Errors { get; set; } /// - /// Workflow id. + /// Is submission deleted. /// - public int WorkflowId { get; set; } + public bool? FilesDeleted { get; set; } + + public SubmissionFile[] InputFiles { get; set; } /// /// Submission input file. @@ -42,15 +129,24 @@ public class Submission : ISubmission /// public string ResultFile { get; set; } + public SubmissionOutput[] OutputFiles { get; set; } + /// /// Is submission retrieved. /// public bool Retrieved { get; set; } - /// - /// Submission errors. - /// - public string Errors { get; set; } + public Review AutoReview { get; set; } + + public SubmissionRetry[] Retries { get; set; } + + public bool? AutoReviewLoaded { get; set; } + + public string OcrEngine { get; set; } + + public Review[] Reviews { get; set; } + + public bool? ReviewInProgress { get; set; } } public class SubmissionSs : Submission @@ -60,14 +156,68 @@ public SubmissionSs(IListSubmissions_Submissions_Submissions submission) { _ssSubmission = submission; Id = _ssSubmission.Id ?? 0; - Status = ConvertFromSs(); DatasetId = _ssSubmission.DatasetId ?? 0; WorkflowId = _ssSubmission.WorkflowId ?? 0; + Status = ConvertFromSs(); + CreatedAt = _ssSubmission.CreatedAt; + UpdatedAt = _ssSubmission.UpdatedAt; + CreatedBy = _ssSubmission.CreatedBy; + UpdatedBy = _ssSubmission.UpdatedBy; + CompletedAt = _ssSubmission.CompletedAt; + Errors = _ssSubmission.Errors; + FilesDeleted = _ssSubmission.FilesDeleted; + InputFiles = _ssSubmission.InputFiles.Select(inputFile => new SubmissionFile + { + Id = inputFile.Id, + FilePath = inputFile.Filepath, + FileName = inputFile.Filename, + FileType = (FileType)inputFile.Filetype, + SubmissionId = inputFile.SubmissionId, + FileSize = inputFile.FileSize, + NumPages = inputFile.NumPages + }).ToArray(); InputFile = _ssSubmission.InputFile; InputFilename = _ssSubmission.InputFilename; ResultFile = _ssSubmission.ResultFile; + OutputFiles = _ssSubmission.OutputFiles.Select(x => new SubmissionOutput() { }).ToArray(); Retrieved = _ssSubmission.Retrieved ?? throw new ArgumentException("Invalid value for retrieved received from call"); - Errors = _ssSubmission.Errors; + AutoReview = _ssSubmission.AutoReview != null ? new Review + { + Id = _ssSubmission.AutoReview.Id, + SubmissionId = _ssSubmission.AutoReview.SubmissionId, + CreatedAt = _ssSubmission.AutoReview.CreatedAt, + CreatedBy = _ssSubmission.AutoReview.CreatedBy, + StartedAt = _ssSubmission.AutoReview.StartedAt, + CompletedAt = _ssSubmission.AutoReview.CompletedAt, + Rejected = _ssSubmission.AutoReview.Rejected, + ReviewType = (ReviewType)_ssSubmission.AutoReview.ReviewType, + Notes = _ssSubmission.AutoReview.Notes, + Changes = _ssSubmission.AutoReview.Changes + } : new Review() { }; + Retries = _ssSubmission.Retries.Select(submissionRetry => new SubmissionRetry + { + Id = submissionRetry.Id, + SubmissionId = submissionRetry.SubmissionId, + PreviousErrors = submissionRetry.PreviousErrors, + PreviousStatus = (SubmissionStatus)submissionRetry.PreviousStatus, + RetryErrors = submissionRetry.RetryErrors + }).ToArray(); + AutoReviewLoaded = _ssSubmission.AutoReviewLoaded; + OcrEngine = _ssSubmission.OcrEngine; + Reviews = _ssSubmission.Reviews.Select(review => new Review + { + Id = review.Id, + SubmissionId = review.SubmissionId, + CreatedAt = review.CreatedAt, + CreatedBy = review.CreatedBy, + StartedAt = review.StartedAt, + CompletedAt = review.CompletedAt, + Rejected = review.Rejected, + ReviewType = (ReviewType)review.ReviewType, + Notes = review.Notes, + Changes = review.Changes + }).ToArray(); + ReviewInProgress = _ssSubmission.ReviewInProgress; } diff --git a/IndicoV2.Abstractions/Submissions/Models/SubmissionFilter.cs b/IndicoV2.Abstractions/Submissions/Models/SubmissionFilter.cs index 75b12a1a..23da99ae 100644 --- a/IndicoV2.Abstractions/Submissions/Models/SubmissionFilter.cs +++ b/IndicoV2.Abstractions/Submissions/Models/SubmissionFilter.cs @@ -1,14 +1,56 @@ -using System; -using System.Collections.Generic; -using System.Dynamic; -using System.Text; - -namespace IndicoV2.Submissions.Models +namespace IndicoV2.Submissions.Models { + public enum FileType + { + Csv, + Doc, + Docx, + Eml, + Excel, + Jpg, + Msg, + Pdf, + Png, + Ppt, + Pptx, + Rtf, + Tiff, + Txt, + Unknown, + Xls, + Xlsx + } + + public enum ReviewType + { + Admin, + Auto, + Manual + } + + public class SubmissionReviewFilter : IFilter + { + public bool? Rejected { get; set; } + public int? CreatedBy { get; set; } + public ReviewType ReviewType { get; set; } + } + + public class DateRangeFilter : IFilter + { + public string From { get; set; } + public string To { get; set; } + } + public class SubmissionFilter : IFilter { + public FileType[] FileType { get; set; } public string InputFilename { get; set; } public SubmissionStatus? Status { get; set; } public bool? Retrieved { get; set; } + public SubmissionReviewFilter Reviews { get; set; } + public bool? ReviewInProgress { get; set; } + public bool? FilesDeleted { get; set; } + public DateRangeFilter CreatedAt { get; set; } + public DateRangeFilter UpdatedAt { get; set; } } } diff --git a/IndicoV2.StrawberryShake/Submissions/Submissions.graphql b/IndicoV2.StrawberryShake/Submissions/Submissions.graphql index 1ff6d491..e0dfed15 100644 --- a/IndicoV2.StrawberryShake/Submissions/Submissions.graphql +++ b/IndicoV2.StrawberryShake/Submissions/Submissions.graphql @@ -21,7 +21,7 @@ mutation WorkflowUrlSubmission( $resultVersion: SubmissionResultVersion ) { workflowUrlSubmission( - workflowId: $workflowId, + workflowId: $workflowId urls: $urls resultVersion: $resultVersion ) { @@ -53,12 +53,67 @@ query ListSubmissions( datasetId workflowId status + createdAt + updatedAt + createdBy + updatedBy + completedAt + errors + filesDeleted + inputFiles { + id + filepath + filename + filetype + submissionId + fileSize + numPages + } inputFile inputFilename resultFile - deleted + outputFiles { + id + filepath + submissionId + componentId + createdAt + } retrieved - errors + autoReview { + id + submissionId + createdAt + createdBy + startedAt + completedAt + rejected + reviewType + notes + changes + } + retries { + id + submissionId + previousErrors + previousStatus + retryErrors + } + AutoReviewLoaded + ocrEngine + reviews { + id + submissionId + createdAt + createdBy + startedAt + completedAt + rejected + reviewType + notes + changes + } + reviewInProgress } pageInfo { startCursor diff --git a/IndicoV2.StrawberryShake/schema.graphql b/IndicoV2.StrawberryShake/schema.graphql index ca2b9224..95b77818 100644 --- a/IndicoV2.StrawberryShake/schema.graphql +++ b/IndicoV2.StrawberryShake/schema.graphql @@ -1,2006 +1,5554 @@ schema { - query: SuperSchema - mutation: SuperMutation - subscription: Subscription + query: Schema + mutation: Mutation +} + +"Directs the executor to include this field or fragment only when the `if` argument is true" +directive @include( + "Included when true." + if: Boolean! +) on FIELD | FRAGMENT_SPREAD | INLINE_FRAGMENT + +"Directs the executor to skip this field or fragment when the `if`'argument is true." +directive @skip( + "Skipped when true." + if: Boolean! +) on FIELD | FRAGMENT_SPREAD | INLINE_FRAGMENT + +"Marks the field, argument, input field or enum value as deprecated" +directive @deprecated( + "The reason for the deprecation" + reason: String = "No longer supported" +) on FIELD_DEFINITION | ARGUMENT_DEFINITION | ENUM_VALUE | INPUT_FIELD_DEFINITION + +"Exposes a URL that specifies the behaviour of this scalar." +directive @specifiedBy( + "The URL that specifies the behaviour of this scalar." + url: String! +) on SCALAR + +"Shared attributes of all components on a workflow" +interface ComponentInterface { + "Actions that can be taken on this component" + actions: [Action] + "Workflow fields that this component can consume or produce" + availableFields: [WorkflowField] + componentAppId: Int + @deprecated( + reason: "This is no longer stored as a first-class property on the component model but in the config for certain types of components" + ) + componentStatus: ComponentStatus @deprecated(reason: "Not yet available") + componentType: ComponentType + "Names of outputs from upstream filters passed to this component" + filteredClasses: [String] + id: Int + "Can the results of this component be used by our Review Interface?" + reviewable: Boolean + "List of valid actions on the component" + validActions: [ValidAction] @deprecated(reason: "Use actions instead") } -type SuperSchema { - allUsers("Include scopes for each user" includeScopes: Boolean "number of pages to skip" skip: Int "Find results before this cursor" before: Int "Find results after this cursor" after: Int "Max number of results to return" limit: Int "Return results in descending order" desc: Boolean "attribute to order results by" orderBy: USER_COLUMN_ENUM filters: UserFilter): UserPage - oneUser(id: Int!): User - userSummary("User summary at this date (23:59 UTC)" date: Date): UserSummary - userSnapshot("Snapshot of permissions at this date (23:59 UTC)" date: Date "number of pages to skip" skip: Int "Find results before this cursor" before: Int "Find results after this cursor" after: Int "Max number of results to return" limit: Int = 100 "Return results in descending order" desc: Boolean "attribute to order results by" orderBy: USERSNAPSHOT_COLUMN_ENUM filters: UserReportFilter): UserSnapshotPage - userChangelog("Get changes on or after this daye (00:00 UTC)" startDate: Date "Get changes on or before this day (23:59 UCT)" endDate: Date "number of pages to skip" skip: Int "Find results before this cursor" before: Int "Find results after this cursor" after: Int "Max number of results to return" limit: Int = 100 "Return results in descending order" desc: Boolean "attribute to order results by" orderBy: USERCHANGELOG_COLUMN_ENUM filters: UserReportFilter): UserChangelogPage - workflows(datasetIds: [Int] workflowIds: [Int] role: Roles metricsStartDate: Date "number of pages to skip" skip: Int "Find results before this cursor" before: Int "Find results after this cursor" after: Int "Max number of results to return" limit: Int "Return results in descending order" desc: Boolean "attribute to order results by" orderBy: WORKFLOW_COLUMN_ENUM filters: WorkflowFilter): WorkflowPage - submission(id: Int!): Submission - checkoutSpecificSubmission(submissionId: Int!): Submission - randomSubmission(workflowId: Int! adminReview: Boolean = false): Submission - submissions(submissionIds: [Int] workflowIds: [Int] "number of pages to skip" skip: Int "Find results before this cursor" before: Int "Find results after this cursor" after: Int "Max number of results to return" limit: Int "Return results in descending order" desc: Boolean "attribute to order results by" orderBy: SUBMISSION_COLUMN_ENUM filters: SubmissionFilter): SubmissionPage - refresh: Refresh - questionnaires(datasetIds: [Int] questionnaireIds: [Int] "number of pages to skip" skip: Int "Find results before this cursor" before: Int "Find results after this cursor" after: Int "Max number of results to return" limit: Int "Return results in descending order" desc: Boolean "attribute to order results by" orderBy: QUESTIONNAIRE_COLUMN_ENUM filters: QuestionnaireFilter): QuestionnairePage - modelGroups(datasetIds: [Int] modelGroupIds: [Int] "number of pages to skip" skip: Int "Find results before this cursor" before: Int "Find results after this cursor" after: Int "Max number of results to return" limit: Int "Return results in descending order" desc: Boolean "attribute to order results by" orderBy: MODELGROUP_COLUMN_ENUM filters: ModelGroupFilter): ModelGroupPage - modelGroup(modelGroupId: Int!): ModelGroup - modelSimilarity(modelGroupId: Int! modelId: Int! query: String!): [ModelSimilarity] - job(id: String): Job - fireGroups(datasetIds: [Int] fireGroupIds: [Int] "number of pages to skip" skip: Int "Find results before this cursor" before: Int "Find results after this cursor" after: Int "Max number of results to return" limit: Int "Return results in descending order" desc: Boolean "attribute to order results by" orderBy: FIREGROUP_COLUMN_ENUM filters: FireGroupFilter): FireGroupPage - fireSimilarity(fireGroupId: Int! fireId: Int! query: String!): [FireSimilarity] - exports(datasetId: [Int] exportIds: [Int] columnIds: [Int] subsetIds: [Int] labelsetIds: [Int] "number of pages to skip" skip: Int "Find results before this cursor" before: Int "Find results after this cursor" after: Int "Max number of results to return" limit: Int "Return results in descending order" desc: Boolean "attribute to order results by" orderBy: EXPORT_COLUMN_ENUM): ExportPage - datafile(datafileId: Int!): DataFile - datafiles(datafileIds: [Int]!): [DataFile] - dataset(id: Int): Dataset - datasets(permissions: [PermissionType]): [Dataset] - datasetsPage(permissions: [PermissionType] showAll: Boolean "number of pages to skip" skip: Int "Find results before this cursor" before: Int "Find results after this cursor" after: Int "Max number of results to return" limit: Int "Return results in descending order" desc: Boolean "attribute to order results by" orderBy: DATASET_COLUMN_ENUM filters: DatasetFilter): DatasetPage - searchDatapoint(datapointId: Int! keyword: String! "Max result length including keyword and surrounding text" context: Int "Use case-insensitive search" ignoreCase: Boolean = true): [DataPointSearchResult] - user: User - ipaVersion: String +"Component representing integrations" +interface IntegrationsInterface { + "Time the integration was created" + createdAt: DateTime + "Whether this integration is currently turned on" + enabled: Boolean @deprecated(reason: "Use Integration.status instead") + "The id of the integration" + id: Int + "The type of integration" + integrationType: IntegrationType + "Details of attempts to communicate with 3rd party" + pagedCommAttempts( + "Find results after this cursor" + after: Int + "Find results before this cursor" + before: Int + "Return results in descending order" + desc: Boolean + "Max number of results to return" + limit: Int = 10 + "attribute to order results by" + orderBy: COMMATTEMPT_COLUMN_ENUM + "number of pages to skip" + skip: Int + ): CommAttemptPage + "Current state of integration" + status: IntegrationStatus + "Id of the associated workflow" + workflowId: Int } -type UserPage { - users: [User] - pageInfo: PageInfo +interface LabelInterface { + datasetuserId: Int + id: Int + rawLabel: [RawLabel] + taskType: TaskType } -"Basic user object" -type User { - id: Int - active: Boolean - acceptedTerms: Boolean - apiRefreshToken: RefreshTokenMeta - "Epoch time of confirmation of user registration" - confirmedAt: String - confirmedDate: DateTime - email: String - lastUpdate: String - lastUpdateDate: DateTime - name: String - "Epoch time of user registration" - registeredAt: String - registeredDate: DateTime - sStreet: String - sStreet2: String - sCity: String - sState: String - sCountry: String - sZip: String - scopes: [ScopeAccess] - preferences(app: String! keys: [String]): [Preference] - logins("number of pages to skip" skip: Int "Find results before this cursor" before: Int "Find results after this cursor" after: Int "Max number of results to return" limit: Int "Return results in descending order" desc: Boolean "attribute to order results by" orderBy: LOGIN_COLUMN_ENUM): LoginPage - accountLockedAt: DateTime - numManagedDatasets: Int +"All OCR options across all engines" +interface OCROptions { + ocrEngine: OCREngine + ocrEngineVersion: String } -type RefreshTokenMeta { +interface PredictionInterface { id: Int - createdAt: DateTime - isApiToken: Boolean - isValid: Boolean - userId: Int + modelId: Int + rawPrediction: [RawPrediction] + taskType: TaskType } -"The `DateTime` scalar type represents a DateTime\nvalue as specified by\n[iso8601](https:\/\/en.wikipedia.org\/wiki\/ISO_8601)." -scalar DateTime +union Evaluation = + AnnotationEvaluation + | ClassificationMultipleEvaluation + | ClassificationSingleEvaluation + | ObjectDetectionEvaluation + | RationalizedClassificationEvaluation + | UnbundlingEvaluation -type ScopeAccess { - scope: Scope - userId: Int +union Prediction = + AnnotationPrediction + | ClassificationMultiplePrediction + | ClassificationPrediction + | RationalizedClassificationPrediction + +type ANNOTATIONTestResult { + falseNegative: [MultiTestResult] + falsePositive: [MultiTestResult] + modelId: Int + resultCounts: ResultCounts + threshold: Float + trueNegative: [MultiTestResult] + truePositive: [MultiTestResult] } -"An enumeration." -enum Scope { - CELERY_FLOWER - MANAGE_USERS - GRAPHIQL - APP_ACCESS - MANAGE_DATASET - MANAGE_ALL_DATA - BASE - REFRESH_TOKEN - CHANGE_PASSWORD - CONFIRM_ACCOUNT - USER_INFORMATION - METRICS +"Represents an action that can be taken on a component or link" +type Action { + "Component Blueprint which can be added, if this is an ADD action" + blueprint: ComponentBlueprint + "Nullable field which specifies the blueprint for ADD actions" + blueprintId: Int + "Object id this action can be performed on" + id: Int + "If this action is not valid, reason for why not. Null if the action is valid" + invalidReason: String + "The specific operation, add/edit/delete" + op: WorkflowActionOp + "Indicates this action can be taken" + valid: Boolean } -type Preference { - app: String - key: String - value: JSONString +type ActiveFormFields { + jobIds: [String] } -"Allows use of a JSON String for input \/ output from the GraphQL schema.\n\nUse of this type is *not recommended* as you lose the benefits of having a defined, static\nschema (one of the key benefits of GraphQL)." -scalar JSONString +type AddDataComponentStatusObject { + color: ComponentStatusColor + status: AddDataComponentStatus + statusMessage: String +} -type LoginPage { - logins: [Login] - pageInfo: PageInfo +type AddDataToWorkflow { + subsetId: Int @deprecated(reason: "No subset id is returned anymore") + workflow: Workflow } -type Login { - id: Int - loggedInAt: DateTimeISO - loginIp: String +"Add a new integration to a workflow. Currently supports exchange." +type AddExchangeIntegration { + integration: ExchangeIntegration } -"ISO-8601 datetime format for JS to parse timezone\n\nFor example: 2020-03-20T01:31:12.467113+00:00\n\nThis custom scalar should only be used for serializing data out, not as an \ninput field, so parse_literal and parse_value are not implemented." -scalar DateTimeISO +type AddModelGroupComponent { + workflow: Workflow +} -type PageInfo { - startCursor: Int - endCursor: Int - hasNextPage: Boolean - aggregateCount: Int +""" +Add more target names to a model group +Field data will be generated if not explicitly opted out of +""" +type AddModelGroupTargetNames { + fields: [WorkflowField] + labelset: LabelSet } -"An enumeration." -enum LOGIN_COLUMN_ENUM { - ID - LOGIN_IP +""" +Add a component of any type to a workflow +'component' can contain: + - component_type: ComponentType, required if not using blueprint + - name: String + - config: JSONString(dict) +""" +type AddWorkflowComponent { + workflow: Workflow } -"An enumeration." -enum USER_COLUMN_ENUM { - ID - ACTIVE - ACCEPTED_TERMS - CONFIRMED_AT - CONFIRMED_DATE - EMAIL - LAST_UPDATE - LAST_UPDATE_DATE - NAME - REGISTERED_AT - REGISTERED_DATE - S_STREET - S_STREET2 - S_CITY - S_STATE - S_COUNTRY - S_ZIP - ACCOUNT_LOCKED_AT - NUM_MANAGED_DATASETS +type AnnotExamplePage { + annotationExamples: [AnnotationExample] + @deprecated(reason: "Please use examples") + examples: [Example] + pageInfo: PageInfo } -input UserFilter { - "name contains" +type AnnotationClassMetrics { + metrics: [AnnotationPerClassSeqMetrics] name: String - "email contains" - email: String - OR: [UserFilter] - AND: [UserFilter] - ors: [UserFilter] - ands: [UserFilter] } -type UserSummary { - users: EnabledCount - appRoles: [AppRoleCount] +type AnnotationEvaluation { + "Query for examples in test set" + examples( + "Find results after this cursor" + after: Int + "Find results before this cursor" + before: Int + "Return results in descending order" + desc: Boolean + filters: ExampleFilter + "Max number of results to return" + limit: Int = 10 + "attribute to order results by" + orderBy: ExampleOrder + "number of pages to skip" + skip: Int + ): AnnotExamplePage + metrics: AnnotationModelMetrics + testResults(actual: [String], threshold: Float): ANNOTATIONTestResult } -type EnabledCount { - "Number of active users" - enabled: Int - "Number of deactivated users" - disabled: Int +type AnnotationExample { + annotationLabels: [AnnotationLabel] + datafile: DataFile + example: Example + id: Int + predictions: [AnnotationPrediction] + rowId: Int + rowIndex: Int + text: String } -type AppRoleCount { - role: AppRole - count: Int +type AnnotationLabel { + clsId: Int + end: Int + label: String + start: Int + userId: Int } -"An enumeration." -enum AppRole { - APP_ADMIN - TEAM_ADMIN - TEAM_DEVELOPER - TEAM_USER - CELERY_FLOWER - MANAGE_ALL_DATA +type AnnotationModelLevelMetrics { + "F1 score calculated per-class and then averaged" + macroF1: Float + "F1 score calculated based on pooling instances across classes" + microF1: Float + "Type of span the metric is calculated on, e.g. Token, Sequence Superset" + spanType: String + "F1 score calculated per-class and then weighted averaged, weighted by instances per class" + weightedF1: Float } -"The `Date` scalar type represents a Date\nvalue as specified by\n[iso8601](https:\/\/en.wikipedia.org\/wiki\/ISO_8601)." -scalar Date +type AnnotationModelMetrics { + "Metrics for evaluating model performance per class" + classMetrics: [AnnotationClassMetrics] + "Metrics for evaluating model performance at the model level, across classes" + modelLevelMetrics: [AnnotationModelLevelMetrics] + "Model retraining is required in order to calculate metrics." + retrainForMetrics: Boolean +} -type UserSnapshotPage { - results: [UserSnapshot] - pageInfo: PageInfo +type AnnotationPerClassSeqMetrics { + "Harmonic mean of precision and recall" + f1Score: Float + "# of examples that were affirmative but were not predicted as such by the model" + falseNegatives: Int + "# of examples that were predicted affirmative in the class but negative" + falsePositives: Int + "Of the predicted true positives, the percentage that were actually correct" + precision: Float + "Of the total true positives, the percentage were recovered by the model as true positives" + recall: Float + "Type of span the metric is calculated on, e.g. Token, Sequence Superset" + spanType: String + "# of examples that were predicted affirmative and were actually affirmative" + truePositives: Int } -type UserSnapshot { - id: Int - name: String - email: String - createdAt: DateTime - enabled: Boolean - roles: [AppRole] - datasets: [DatasetRole] +type AnnotationPrediction { + clsId: Int + confidence: Float + end: Int + label: String + start: Int + text: String } -type DatasetRole { - datasetId: Int - role: Roles +type AppRoleCount { + count: Int + role: AppRole } -"An enumeration." -enum Roles { - LABELER - REVIEWER - LABELER_AND_REVIEWER - ANALYST - MANAGER +""" +Represents a possible action on a blueprint, +and whether it's valid or not +""" +type BPAction { + invalidReason: String + op: BlueprintOP + valid: Boolean } -"An enumeration." -enum USERSNAPSHOT_COLUMN_ENUM { - ID - NAME - EMAIL - CREATED_AT - ENABLED +"Blueprints have a standard tag and value for display" +type BTag { + tag: BlueprintTag + value: String } -input UserReportFilter { - "User id in this list" - userId: [Int] - "User email in this list" - userEmail: [String] - OR: [UserReportFilter] - AND: [UserReportFilter] - ors: [UserReportFilter] - ands: [UserReportFilter] +type BaseTargetName { + name: String } -type UserChangelogPage { - results: [UserChangelog] - pageInfo: PageInfo +"A workflow associated with a particular blueprint" +type BlueprintAssociatedWorkflow { + "Id of the dataset associated with a paricular blueprint" + datasetId: Int + "Id of the workflow" + id: Int + "Name of the workflow" + name: String } -type UserChangelog { - "Unique combination of date and user_id" - id: String - date: DateTime +"A pagination of workflows associated with a particular blueprint, and workflows available to the user" +type BlueprintAssociatedWorkflowPage { + "List of workflows for a given blueprint that are accessible to the user" + linkableWorkflowIds: [Int] + "page information for blueprint associated workflows" + pageInfo: PageInfo + "List of workflows associated with a blueprint" + workflows: [BlueprintAssociatedWorkflow] +} + +""" +Component subclass for a Component created using a "blueprint" task +Deprecated! Not to be confused with ComponentBlueprints +""" +type BlueprintComponent implements ComponentInterface { + "Actions that can be taken on this component" + actions: [Action] + "Workflow fields that this component can consume or produce" + availableFields: [WorkflowField] + blueprintId: String + componentAppId: Int + @deprecated( + reason: "This is no longer stored as a first-class property on the component model but in the config for certain types of components" + ) + componentStatus: ComponentStatus @deprecated(reason: "Not yet available") + componentType: ComponentType + "Names of outputs from upstream filters passed to this component" + filteredClasses: [String] + id: Int + "Can the results of this component be used by our Review Interface?" + reviewable: Boolean + "List of valid actions on the component" + validActions: [ValidAction] @deprecated(reason: "Use actions instead") +} + +type CLASSIFICATIONTestResult { + falseNegative: [SingleTestResult] + falsePositive: [SingleTestResult] + modelId: Int + resultCounts: ResultCounts + threshold: Float + trueNegative: [SingleTestResult] + truePositive: [SingleTestResult] +} + +type CLASSIFICATION_MULTIPLETestResult { + falseNegative: [MultiTestResult] + falsePositive: [MultiTestResult] + modelId: Int + resultCounts: ResultCounts + threshold: Float + trueNegative: [MultiTestResult] + truePositive: [MultiTestResult] +} + +type CLASSIFICATION_UNBUNDLINGTestResult { + falseNegative: [MultiTestResult] + falsePositive: [MultiTestResult] + modelId: Int + resultCounts: ResultCounts + threshold: Float + trueNegative: [MultiTestResult] + truePositive: [MultiTestResult] +} + +type CancelModelTraining { + success: Boolean +} + +type ClassBalance { + all: [ClassCount] + majorityVoteWithTies: [ClassCount] + majorityVoteWithoutTies: [ClassCount] + unanimous: [ClassCount] +} + +type ClassConfidence { + confidence: Float + name: String +} + +type ClassConfidenceV2 { + clsName: TargetName + confidence: Float +} + +type ClassCount { + count: Int + target: String +} + +type ClassLabel implements LabelInterface { + datasetuserId: Int + id: Int + label: _ClassLabel + rawLabel: [RawLabel] + taskType: TaskType +} + +type ClassPrediction implements PredictionInterface { + id: Int + modelId: Int + prediction: _ClassPred + rawPrediction: [RawPrediction] + taskType: TaskType +} + +type ClassStpFacts { + "Auto Review STP for class aggregated by parent filter context" + autoReviewStpForClass: Float + className: String + "Review STP for class aggregated by parent filter context." + reviewStpForClass: Float + stps: [DailyStp] +} + +type ClassStpMetrics { + "STP metrics for this class, aggregated based on the filters applied to the query" + aggregate: StpMetric + className: String + "STP metrics for this class, daily" + daily: [DailyStpMetric] +} + +type ClassificationModelMetrics { + confusionMatrix: ConfusionMatrix + metricsTable: [MetricsTable] + prCurves: [PRCurve] + rocCurves: [ROCCurve] +} + +type ClassificationMultipleEvaluation { + confusionResult(actual: String!, predicted: String!): [SingleTestResult] + metrics: ClassificationModelMetrics + testResults( + actual: [String] + threshold: Float + ): CLASSIFICATION_MULTIPLETestResult +} + +type ClassificationMultiplePrediction { + confidences: [ClassConfidence] + explanations: [Explanation] + labels: [String] + tokenPredictions: [TokenPrediction] +} + +type ClassificationPrediction { + confidences: [ClassConfidence] + explanations: [Explanation] + label: String + tokenPredictions: [TokenPrediction] +} + +type ClassificationSingleEvaluation { + confusionResult(actual: String!, predicted: String!): [SingleTestResult] + metrics: ClassificationModelMetrics + testResults(actual: String, threshold: Float): CLASSIFICATIONTestResult +} + +type CommAttempt { + "If attempt failed, reason for failure" + error: String + "When indico finished communicating with 3rd party" + finished: DateTime + id: Int + "When communication attempt started" + started: DateTime + "Whether the communication attempt was successful" + success: Boolean +} + +type CommAttemptPage { + commAttempts: [CommAttempt] + pageInfo: PageInfo +} + +"A single component of a workflow, usually based on some blueprint" +type Component implements ComponentInterface { + "Actions that can be taken on this component" + actions: [Action] + "Workflow fields that this component can consume or produce" + availableFields: [WorkflowField] + componentAppId: Int + @deprecated( + reason: "This is no longer stored as a first-class property on the component model but in the config for certain types of components" + ) + componentStatus: ComponentStatus @deprecated(reason: "Not yet available") + componentType: ComponentType + "Names of outputs from upstream filters passed to this component" + filteredClasses: [String] + id: Int + "Name of component" + name: String + "Can the results of this component be used by our Review Interface?" + reviewable: Boolean + "List of valid actions on the component" + validActions: [ValidAction] @deprecated(reason: "Use actions instead") +} + +""" +A component blueprint represents a template from which +workflow components can be created. +""" +type ComponentBlueprint { + "Actions that can be taken on the blueprint" + actions: [BPAction] + "Blueprint belongs to this family" + componentFamily: ComponentFamily + "Blueprint is of this type" + componentType: ComponentType + "ComponentBlueprint config" + config: GenericScalar + "Description for this blueprint" + description: String + "This blueprint is enabled on this platform" + enabled: Boolean + "Fields this blueprint outputs" + fields: [FieldData] + "Footnotes for this blueprint's description" + footer: String + "Icon representing this blueprint" + icon: String + "Id of the blueprint which can be used to create components" + id: Int + "Name of blueprint" + name: String + "Provider responsible for developing this blueprint" + provider: String + "Tags for this component" + tags: [BlueprintTag] + "List of workflows for a given blueprint" + usedWorkflows( + "Find results after this cursor" + after: Int + "Find results before this cursor" + before: Int + "Return results in descending order" + desc: Boolean + "Max number of results to return" + limit: Int + "attribute to order results by" + orderBy: BLUEPRINTASSOCIATEDWORKFLOW_COLUMN_ENUM + "number of pages to skip" + skip: Int + ): BlueprintAssociatedWorkflowPage +} + +type ComponentBlueprintPage { + componentBlueprints: [ComponentBlueprint] + pageInfo: PageInfo +} + +type ComponentGallery { + "All tags available for component blueprints. Optionally filtered by family" + availableTags( + "Specific tags for a component family" + componentFamily: ComponentFamily + ): [BTag] + "Blueprint by id" + blueprint(id: Int!): ComponentBlueprint + "Blueprints by ids" + blueprints(ids: [Int]!): [ComponentBlueprint] + blueprintsPage( + "Find results after this cursor" + after: Int + "Find results before this cursor" + before: Int + "Return results in descending order" + desc: Boolean + filters: ComponentBlueprintFilter + "Max number of results to return" + limit: Int + "attribute to order results by" + orderBy: COMPONENTBLUEPRINT_COLUMN_ENUM + "number of pages to skip" + skip: Int + ): ComponentBlueprintPage +} + +""" +Object which represents the one-way connection between 2 components +in a workflow +""" +type ComponentLink { + "Actions that can be taken on this link" + actions: [Action] + "Workflow fields that this component link consumes" + availableFields: [WorkflowField] + "Base configuration for this link" + config: JSONString + "Filters applied to documents flowing on this link" + filters: Filters + "Parent component documents flow from" + headComponentId: Int + id: Int + "Child component documents flow to" + tailComponentId: Int + "List of valid actions on the component link" + validActions: [ValidAction] @deprecated(reason: "Use actions instead") + "Workflow this link belongs to" + workflowId: Int +} + +type ComponentStatus { + addData: AddDataComponentStatusObject + submission: SubmissionComponentStatusObject +} + +type ConfusionMatrix { + classes: [String] + matrix: [[Float]] +} + +"Component subclass for a Content-length Filter Component" +type ContentLengthComponent implements ComponentInterface { + "Actions that can be taken on this component" + actions: [Action] + "Workflow fields that this component can consume or produce" + availableFields: [WorkflowField] + componentAppId: Int + @deprecated( + reason: "This is no longer stored as a first-class property on the component model but in the config for certain types of components" + ) + componentStatus: ComponentStatus @deprecated(reason: "Not yet available") + componentType: ComponentType + "Names of outputs from upstream filters passed to this component" + filteredClasses: [String] + id: Int + maximum: Int + minimum: Int + "Name of component" + name: String + "Can the results of this component be used by our Review Interface?" + reviewable: Boolean + "List of valid actions on the component" + validActions: [ValidAction] @deprecated(reason: "Use actions instead") +} + +type CreateWorkflow { + workflow: Workflow +} + +type CreateWorkflowFromBlueprint { + workflow: Workflow +} + +"Component subclass for a Custom Deployed Component" +type CustomComponent implements ComponentInterface { + "Actions that can be taken on this component" + actions: [Action] + "Workflow fields that this component can consume or produce" + availableFields: [WorkflowField] + componentAppId: Int + @deprecated( + reason: "This is no longer stored as a first-class property on the component model but in the config for certain types of components" + ) + componentStatus: ComponentStatus @deprecated(reason: "Not yet available") + componentType: ComponentType + "Names of outputs from upstream filters passed to this component" + filteredClasses: [String] + id: Int + "Name of custom component" + name: String + "Can the results of this component be used by our Review Interface?" + reviewable: Boolean + "List of valid actions on the component" + validActions: [ValidAction] @deprecated(reason: "Use actions instead") +} + +"Component subclass for a Custom Deployed Model" +type CustomModelComponent implements ComponentInterface { + "Actions that can be taken on this component" + actions: [Action] + "Workflow fields that this component can consume or produce" + availableFields: [WorkflowField] + "Output class names" + classNames: [BaseTargetName] + componentAppId: Int + @deprecated( + reason: "This is no longer stored as a first-class property on the component model but in the config for certain types of components" + ) + componentStatus: ComponentStatus @deprecated(reason: "Not yet available") + componentType: ComponentType + "Names of outputs from upstream filters passed to this component" + filteredClasses: [String] + id: Int + "Name of custom component" + name: String + "Can the results of this component be used by our Review Interface?" + reviewable: Boolean + "List of valid actions on the component" + validActions: [ValidAction] @deprecated(reason: "Use actions instead") +} + +type DailyAvg { + avg: Float + date: Date +} + +type DailyCount { + count: Int + date: Date +} + +type DailyPredictionMetric { + date: Date + numPreds: Int +} + +type DailyQueueMetric { + "Average cumulative age of items on queues waiting to be reviewed [QUEUES_STAT]" + avgAgeInQueue: Float + date: Date + "Cumulative hours of items on queues waiting to be reviewed [QUEUES_STAT]" + hoursOnQueue: Float + "Number of submissions on the queues waiting to be reviewed [QUEUES_STAT]" + subsOnQueue: Int +} + +type DailyStp { + autoReviewDenom: Int + autoReviewNumerator: Int + autoReviewStpPct: Float + date: Date + reviewDenom: Int + reviewNumerator: Int + reviewStpPct: Float +} + +type DailyStpMetric { + "The union of user supplied labels and auto review labels [PREDICTIONS_STP]" + autoReviewDenom: Int + "The number of human accepted auto review labels [PREDICTIONS_STP]" + autoReviewNumerator: Int + "Auto review numerator divided by auto review denomoinator, applicable if auto-review is enabled [PREDICTIONS_STP]" + autoReviewStpPct: Float + date: Date + "The union of user supplied labels and model predictions [PREDICTIONS_STP]" + reviewDenom: Int + "The number of human accepted model predictions that exactly match model predictions [PREDICTIONS_STP]" + reviewNumerator: Int + "Review numerator divided by review denominator, applicable if review is enabled and auto-review is disabled [PREDICTIONS_STP]" + reviewStpPct: Float +} + +type DailySubmissionMetric { + "Number of items completed in the workflow, whether review was enabled or disabled [SUBMISSION_COMPLETED]" + completed: Int + "Number of items accepted in the exceptions queue [SUBMISSION_REVIEW_STAT]" + completedExceptionQueue: Int + "Number of items that were accepted in either the review or exceptions queue [SUBMISSION_REVIEW_STAT]" + completedInReview: Int + "Number of items accepted in the review queue [SUBMISSION_REVIEW_STAT]" + completedReviewQueue: Int + date: Date + "Number of items rejected from the exceptions queue [SUBMISSION_REVIEW_STAT]" + rejectedInReview: Int + "Number of items submitted to the workflow [SUBMISSION_SUBMITTED]" + submitted: Int +} + +type DailyTimeOnTask { + date: Date + minutes: Float + numReviews: Int +} + +type DailyTimeOnTaskMetric { + "The average amount of minutes reviewers spend on documents for this workflow, aggregated across review and exceptions queue" + avgMinsPerDoc: Float + "The average amount of minutes reviewers spend on documents for this workflow in the exceptions queue" + avgMinsPerDocExceptions: Float + "The average amount of minutes reviewers spend on documents for this workflow in the review queue" + avgMinsPerDocReview: Float + date: Date +} + +""" +Represents the columns of data in a +dataset that can be used for training +If the dataset is created using CSVs, the name +comes from column name in the CSV. If a CSV is not used +or if the CSV column name is blank, a default name is created. +For example, if your dataset is just links to files, the column +is often called 'documents' +""" +type DataColumn { + "order of the column in the dataset. Usually relevant with CSVs" + columnIndex: Int + "id of the dataset this column belongs to" + datasetId: Int + "type of data included in the column" + datatype: DataTypes + id: Int + "name of this column" + name: String + "was OCR used to parse the data in the column" + ocrUsed: Boolean +} + +"Data processing configurations for dataset" +type DataConfig { + emailOptions: EmailOptions + ocrOptions: OCROptions +} + +""" +Represents 1 source file within a Dataset +May be a CSV, which other DataFiles can spawn from +or an image or pdf, etc, which is what models and workflows +will use for training data +""" +type DataFile { + "This datafile was deleted from the dataset" + deleted: Boolean + "If this Datafile failed processing, this is why" + failureType: FileFailureType + "Redundant representation of the rainbow_url" + fileHash: String + "Size of the datafile in bytes" + fileSize: Int + "Type of the datafile - UNKNOWN if not processed yet" + fileType: FileType + id: Int + "Name of the uploaded file" + name: String + "Number of pages in this datafile" + numPages: Int + "Ids of the datafilepages extracted from this datafile, if any" + pageIds: [Int] + "Page objects extracted from this datafile" + pages(pageNums: [Int]): [DataFilePage] + "Indico URL to the source file" + rainbowUrl: String + "Status of the file as it's being processed for the dataset" + status: FileStatus + "More information about the current status, including failure information" + statusMeta: JSONString +} + +"A single page within a source datafile" +type DataFilePage { + "The datafile this page belongs to" + datafileId: Int + "Ending char index of this page in the context of the datafile" + docEndOffset: Int + "Starting char index of this page in the context of the datafile" + docStartOffset: Int + id: Int + "Indico URL to an image of this page" + image: String + "Indico URL to a JSON file containing OCR data for this datafile" + pageInfo: String + "Index of this page within the source datafile" + pageNum: Int + "Indico URL to a thumbnail image of this page" + thumbnail: String +} + +""" +Cyclone Label Breakdown Response + +response = { + "num_empty_examples": len(empty_sources), + "num_examples": num_examples, + "labelset_id": labelset.id, + "target_contradictions": percent_contradictions, + "source_duplicates": percent_source_duplicate, + "class_balance": target_counter, + "warnings": warnings, +} +""" +type DataInfo { + classBalance: ClassBalance + datasetId: Int + labelsetId: Int + numEmptyExamples: Int + numExamples: Int + sourceColumnId: Int + sourceDuplicates: Float + subsetIds: [Int] @deprecated(reason: "No longer used") + targetContradictions: Float + warnings: [String] +} + +""" +This is the beginning of working with Indico! A dataset +represents all the possible training data you'd like to work with +to build one or more workflows. Includes users you'd like to work +on the dataset with, the source datafiles, any labelsets for the files, +etc +""" +type Dataset { + "When your dataset was first created" + createdAt: String + "User id of the user that created this dataset" + createdBy: Int + "Configuration used to parse and extract information from the datafiles added to this dataset. The same configuration will be used to parse and extract information from files submitted to workflows on this dataset" + dataConfig: DataConfig + dataInfo(labelsetId: Int, sourceColumnId: Int): DataInfo + @deprecated(reason: "Fetch class balance info under ModelGroup") + "All the source Datacolumns in this dataset. Does NOT include any label columns uploaded as part of the dataset with CSV datafiles" + datacolumns: [DataColumn] + "Id of the datacolumn that will be used to source training data for all of your models and workflows. See DataColumn for more info" + defaultDatacolumnId: Int + defaultSubsetId: Int + @deprecated(reason: "No longer use subsets to limit data") + "Any errors encountered while creating your dataset" + errorInfo: String + "Data exports made for this dataset" + exports: [Export] + "Types of files in this dataset" + fileTypes( + "Include only filetypes associated with rows" + filterable: Boolean = true + ): [FileType] + "All the source files added to this dataset" + files: [DataFile] + id: Int + "All the source label columns in this dataset" + labelsets: [LabelSet] + "Retrieve a specific Model Group belonging to this dataset by id" + modelGroup(id: Int): ModelGroup + "All model groups that were created from this dataset" + modelGroups(taskTypes: [TaskType]): [ModelGroup] + "Name of this dataset" + name: String + "Number of model groups belonging to this Dataset" + numModelGroups: Int + "Number of questionnaires aka Teach Tasks belonging to this Dataset" + numQuestionnaires: Int + "Permissions the current user has on this dataset" + permissions: [String] + "Number of rows in the dataset that can be used in model training" + rowCount: Int + "Status of this dataset which allows it to be used to create workflows, etc" + status: DatasetStatus + "Type of data your dataset holds" + type: DatasetType + "The last time properties of the dataset changed" + updatedAt: String + "User id of the user that edited or caused dataset changes" + updatedBy: Int + "All the users that have explicit access to this dataset" + users: [DatasetUser] +} + +"Paginated result object for listing datasets" +type DatasetPage { + datasets: [Dataset] + "Information about this current page. See PageInfo for more information" + pageInfo: PageInfo +} + +type DatasetRole { + datasetId: Int + role: Roles +} + +""" +A user on a dataset. References a Platform CoreUser +but with dataset-specific permissions +""" +type DatasetUser { + "Id of dataset this user belongs to" + datasetId: Int + "Email of CoreUser" + email: String + id: Int + "Name of CoreUser" + name: String + "Dataset permissions this user has on the dataset" + permissions: [String] + "Dataset role (aggregate of permissions) this user has on the dataset" + role: Roles + "Platform CoreUser reference" userId: Int - userEmail: String +} + +"Delete this dataset from the platform" +type DeleteDataset { + success: Boolean +} + +"Remove a user from a dataset. Also remove the user from all dataset tasks" +type DeleteDatasetUser { + success: Boolean +} + +""" +Deletes an integration from a workflow. This permanently removes the integration. +You cannot undo this action. +""" +type DeleteIntegration { + "True if the integration is succesfully deleted" + success: Boolean +} + +type DeletePreference { + success: Boolean +} + +type DeleteWorkflow { + success: Boolean +} + +type DeleteWorkflowComponent { + workflow: Workflow +} + +""" +Extract text from a document using one of our OCR engine providers. +Returns: List of ids of the document processing jobs kicked off by this mutation. Can be used to query for status and results. +""" +type DocumentExtraction { + jobIds: [String] +} + +"Email options for dataset" +type EmailOptions { + includeSections: EmailSectionOptions +} + +"Email sections to include for processing" +type EmailSectionOptions { + attachments: Boolean + body: Boolean + header: Boolean +} + +type EnabledCount { + "Number of deactivated users" + disabled: Int + "Number of active users" + enabled: Int +} + +"Generic class to hold OCR Engine configuration fields" +type EngineConfigField { + description: String + fieldType: String + name: String +} + +"Holds all possible configurations for the OCR Engine" +type EngineOptions { + defaultLanguage: OCRLanguage + formattedName: String + languages: [OCRLanguage] + legacy: Boolean + name: OCREngine + options: [EngineConfigField] + typeName: String + versionName: String +} + +""" +An object that tracks labels and predictions for some +piece of data on a labelset +Use sub-classes so in the future, different queries can +define which exact examples they supply +ie example list - no labels or preds +relabel - w/ labels and preds +teach - w/ preds +""" +type Example { + autolabeled: Boolean + context: ExampleContext + datafile: DataFile + datafileId: Int + datapointId: Int + datarowId: Int + id: Int + labels: [LabelInterface] + labelsetId: Int + modelId: Int + modelPrediction: PredictionInterface + partial: Boolean + rowIndex: Int @deprecated(reason: "No longer unique. Please use id") + "Search for text within example" + search( + "Max result length including keyword and surrounding text" + context: Int + "Use case-insensitive search" + ignoreCase: Boolean = true + "Keyword to search the text for" + keyword: String! + "Use a regex keyword to find matches in the text" + regex: Boolean + ): [TextSearchResult] + "HIGH-COST: url or text of contextualized example" + source: String + status: ExampleStatus + taskType: TaskType + updatedAt: DateTime +} + +""" +The portion of the source file that this example +will cover +""" +type ExampleContext { + "pixel bounds of the image" + bounds: [SpatialSpan] + id: Int + "character bounds of the text" + spans: [TokenSpan] +} + +type ExamplePage { + examples: [Example] + pageInfo: PageInfo +} + +type ExchangeIntegration implements IntegrationsInterface { + config: ExchangeIntegrationConfiguration + "Time the integration was created" + createdAt: DateTime + "Whether this integration is currently turned on" + enabled: Boolean @deprecated(reason: "Use Integration.status instead") + "The id of the integration" + id: Int + "The type of integration" + integrationType: IntegrationType + "Details of attempts to communicate with 3rd party" + pagedCommAttempts( + "Find results after this cursor" + after: Int + "Find results before this cursor" + before: Int + "Return results in descending order" + desc: Boolean + "Max number of results to return" + limit: Int = 10 + "attribute to order results by" + orderBy: COMMATTEMPT_COLUMN_ENUM + "number of pages to skip" + skip: Int + ): CommAttemptPage + "Current state of integration" + status: IntegrationStatus + "Id of the associated workflow" + workflowId: Int +} + +type ExchangeIntegrationConfiguration { + "Filters for inbox ie: is_read, is_from_domain, is_unread, is_not_from_domain, etc. " + filters: JSONString + "Identifier for mailbox folder" + folderId: String + "Display name of the mailbox folder" + folderName: String + "User's ID" + userId: String +} + +type ExplainMeta { + datasetId: Int + exampleId: Int + rowId: Int @deprecated(reason: "Please use example id") + rowIndex: Int @deprecated(reason: "Please use example id") + sourceColumnId: Int +} + +type Explanation { + label: String + metadata: ExplainMeta + similarity: Float + text: String +} + +type Export { + "Indicates user ids have been anonymized" + anonymous: Boolean + columnIds: [Int] + "Unix timestamp" + createdAt: String + "user id" + createdBy: Int + datasetId: Int + "Download URL of this export" + downloadUrl: String + "Frozen labelsets to select examples" + frozenLabelsetIds: [Int] + id: Int + labelsetId: Int + labelsetIds: [Int] @deprecated(reason: "Use labelset id") + modelIds: [Int] + name: String + "Number of labels in the export" + numLabels: Int + status: ExportStatus + subsetIds: [Int] @deprecated(reason: "Use frozen_labelset_ids") +} + +type ExportPage { + exports: [Export] + pageInfo: PageInfo +} + +"Component subclass for an OOTB, Third-Party Model" +type ExternalModelComponent implements ComponentInterface { + "Actions that can be taken on this component" + actions: [Action] + "Workflow fields that this component can consume or produce" + availableFields: [WorkflowField] + "Output class names" + classNames: [BaseTargetName] + componentAppId: Int + @deprecated( + reason: "This is no longer stored as a first-class property on the component model but in the config for certain types of components" + ) + componentStatus: ComponentStatus @deprecated(reason: "Not yet available") + componentType: ComponentType + "Names of outputs from upstream filters passed to this component" + filteredClasses: [String] + id: Int + "Name of external model service" + modelService: String + "Name of external model" + name: String + "Can the results of this component be used by our Review Interface?" + reviewable: Boolean + taskType: TaskType + "List of valid actions on the component" + validActions: [ValidAction] @deprecated(reason: "Use actions instead") +} + +type ExtractLabel implements LabelInterface { + datasetuserId: Int + id: Int + label: [_ExtractLabel] + rawLabel: [RawLabel] + taskType: TaskType +} + +type ExtractPrediction implements PredictionInterface { + id: Int + modelId: Int + prediction: [_ExtractionPred] + rawPrediction: [RawPrediction] + taskType: TaskType +} + +"Represents a field output on a blueprint" +type FieldData { + "Data type of the field" + datatype: String + "Icon to display for the field" + icon: String + "Name of the field" + name: String +} + +""" +Definition for which filters have been applied between +a parent and child component +""" +type Filters { + "Label/Prediction filters for the documents" + classes: [String] + @deprecated(reason: "Classes are populated from workflow field options") + "Content length filter requirements" + length: LengthFilter + "This link goes straight from the component to output" + passed: Boolean +} + +"Object with labelset snapshot information for when the model is trained" +type FrozenLabelSet { + "The target names of the model at the time of training" + frozenTargetNames: [TargetName] + "ID of the labelset snapshot" + id: Int +} + +"Representing the full gallery for users" +type Gallery { + "Gallery for components" + component: ComponentGallery + "Gallery for workflows" + workflow: WorkflowGallery +} + +""" +Example: +mutation generate_new_refresh_token { + GenerateNewApiRefreshToken{ + refresh_token + } +} +""" +type GenerateNewApiRefreshToken { + refreshToken: String +} + +type GenerateResetLink { + link: String +} + +type GenerateSubmissionReport { + jobId: String +} + +type GenerateUserChangelogReport { + jobId: String +} + +type GenerateUserSnapshotReport { + jobId: String +} + +""" +Subclass for a component representing the initial data collection and OCR +state of a workflow +""" +type InputComponent implements ComponentInterface { + "Actions that can be taken on this component" + actions: [Action] + "Workflow fields that this component can consume or produce" + availableFields: [WorkflowField] + componentAppId: Int + @deprecated( + reason: "This is no longer stored as a first-class property on the component model but in the config for certain types of components" + ) + componentStatus: ComponentStatus @deprecated(reason: "Not yet available") + componentType: ComponentType + "Names of outputs from upstream filters passed to this component" + filteredClasses: [String] + id: Int + "List of integrations created for this workflow" + integrations: [IntegrationsInterface] + "Can the results of this component be used by our Review Interface?" + reviewable: Boolean + "List of valid actions on the component" + validActions: [ValidAction] @deprecated(reason: "Use actions instead") +} + +type Job { + id: String + ready: Boolean + result: JSONString + status: JobStatus +} + +type LabelSet { + datacolumnId: Int + datasetId: Int + id: Int + name: String + numFullyLabeled: Int + numLabelersRequired: Int + numLabelsetPoints: Int @deprecated(reason: "Use total_num_examples instead") + numRejected: Int + "The source data for this labelset was produced from an upstream model's labels" + spanned: Boolean + targetNames: [TargetName] + "Target types available for this labelset's task type" + targetTypes: [TargetType] + taskType: TaskType + "Number of examples in labelset" + totalNumExamples: Int +} + +""" +Object describing the differences between the current labelset and its prior +state when training occurred +""" +type LabelSetDiff { + "Lists of added and removed target names from the set the model was trained on" + targetNames: TargetNameDiff +} + +""" +Definition of a content length filtering strategy +for a Content Length Filter Component +""" +type LengthFilter { + description: String + maximum: Int + minimum: Int +} + +"Component subclass for a Link component that follows a classification-task-type model component" +type LinkClassificationModelComponent implements ComponentInterface { + "Actions that can be taken on this component" + actions: [Action] + "Workflow fields that this component can consume or produce" + availableFields: [WorkflowField] + "Selected classes from the classification model linked" + classes: [[String]] + @deprecated(reason: "Classes are populated from workflow field options") + componentAppId: Int + @deprecated( + reason: "This is no longer stored as a first-class property on the component model but in the config for certain types of components" + ) + componentStatus: ComponentStatus @deprecated(reason: "Not yet available") + componentType: ComponentType + "Names of outputs from upstream filters passed to this component" + filteredClasses: [String] + id: Int + "Predicted or actual" + labels: String @deprecated(reason: "Actual labels always used") + "Name of component" + name: String + "Can the results of this component be used by our Review Interface?" + reviewable: Boolean + "List of valid actions on the component" + validActions: [ValidAction] @deprecated(reason: "Use actions instead") +} + +""" +Component subclass for a Component which follows an extraction-task-type +Model Component and groups predictions together into user-defined groups +""" +type LinkLabelComponent implements ComponentInterface { + "Actions that can be taken on this component" + actions: [Action] + "Workflow fields that this component can consume or produce" + availableFields: [WorkflowField] + componentAppId: Int + @deprecated( + reason: "This is no longer stored as a first-class property on the component model but in the config for certain types of components" + ) + componentStatus: ComponentStatus @deprecated(reason: "Not yet available") + componentType: ComponentType + "Names of outputs from upstream filters passed to this component" + filteredClasses: [String] + "configuration for a single group" + groups: [LinkLabelGroup] + id: Int + "ID of the labelset that the combined labels come from" + labelsetId: Int + @deprecated(reason: "Obsolete. Refer to the workflow's fields instead.") + "Model group that outputs the labels to be combined" + modelGroupId: Int + "Name of component" + name: String + "Can the results of this component be used by our Review Interface?" + reviewable: Boolean + "List of valid actions on the component" + validActions: [ValidAction] @deprecated(reason: "Use actions instead") +} + +"Definition for a \"grouping strategy\" to be used by a Link Label Component" +type LinkLabelGroup { + "id of the model group that this component comes after" + ModelGroupId: Int + "ids and names for classes to be included in the group" + classNameValues: [TargetName] @deprecated(reason: "Reference fields instead.") + "IDs for classes to be included in the group" + classNames: [Int] @deprecated(reason: "Reference field ids instead.") + "IDs for workflow fields to be included in the group." + fieldIds: [Int] + "Workflow fields included in the group" + fields: [WorkflowField] + "unique id for this group" + id: String + "name for this group" + name: String + "strategy for merging labels into groups" + strategy: LabelLinkStrategy + "strategy-specific settings for how labels are combined" + strategySettings: JSONString +} + +type Login { + id: Int + loggedInAt: DateTimeISO + loginIp: String +} + +type LoginPage { + logins: [Login] + pageInfo: PageInfo +} + +type MetricsTable { + accuracy: Float + auc: Float + f1Score: Float + name: String + precision: Float + recall: Float +} + +type Model { + classNames: [String] + collectionName: String + createdAt: String + createdBy: Int + evaluation: Evaluation + id: Int + "Changes made to the labelset since the model was last trained, compared against the frozen state of the model's labelset. In the case of missing frozen labelsets, none is returned" + labelsetDiff: LabelSetDiff + linkable: Boolean + "File path to the model" + modelFilePath: String + modelGroupId: Int + modelInfo: JSONString + modelOptions: JSONString + modelType: ModelType + predictionLabelsetId: Int + predictions(sources: [String], threshold: Float): [Prediction] + rareClasses: [ClassCount] + status: ModelStatus + "Testing labelset snapshot information for when the model is trained" + testingFrozenLabelset: FrozenLabelSet + testingFrozenLabelsetId: Int + testingSubsetId: Int @deprecated(reason: "Use testing_frozen_labelset_id") + "Training labelset snapshot information for when the model is trained" + trainingFrozenLabelset: FrozenLabelSet + trainingFrozenLabelsetId: Int + trainingProgress: TrainingProgress + trainingSubsetId: Int @deprecated(reason: "Use training_frozen_labelset_id") + unlabeledClasses: [String] + updatedAt: String updatedBy: Int - updaterEmail: String - previouslyEnabled: Boolean - enabled: Boolean - previousRoles: [AppRole] - roles: [AppRole] - previousDatasets: [DatasetRole] - datasets: [DatasetRole] - changesMade: [UserChangeType] } -"An enumeration." -enum UserChangeType { - APP_ROLE - DATASET_ROLE - ENABLEMENT +type ModelGroup { + "Ordered list of target names used to train the model." + classNames: [String] @deprecated(reason: "Please use clsNames instead") + "ID of the workflow component associated with this model group." + componentId: Int + "Time of model group creation" + createdAt: String + "User ID that created this model gruop" + createdBy: Int + dataInfo: DataInfo + dataType: DataTypes + datasetId: Int + "Query for a single example with any labels and selected model predictions" + example( + exampleId: Int + "Deprecated - please specify example_id" + rowIndex: Int + ): Example + "All field links associated with this component ID." + fieldLinks: [WorkflowFieldLink] + "List of fields associated with this particular model" + fields: [WorkflowField] + id: Int + interlabelerResolution: LabelResolutionStrategy + labelset: LabelSet + "Column id where model labels are stored" + labelsetColumnId: Int + model(id: Int!): Model + modelOptions: ModelOptions + models: [Model] + name: String + "Query for example list page. Examples do not provide labels or predictions" + pagedExamples( + "Find results after this cursor" + after: Int + "Find results before this cursor" + before: Int + "Return results in descending order" + desc: Boolean + filters: ExampleFilter + "Max number of results to return" + limit: Int = 10 + "attribute to order results by" + orderBy: ExampleOrder + "number of pages to skip" + skip: Int + ): ExamplePage + processors: [Processor] + "ID of the questionnaire this model group was created from" + questionnaireId: Int + retrainRequired: Boolean + selectedModel: Model + selectedModelId: Int + sourceColumn: DataColumn + "Column id where raw data for model is stored" + sourceColumnId: Int + status: ModelStatus + "ID of the data subset this model group was created from" + subsetId: Int + taskType: TaskType + "Time of last model group update" + updatedAt: String + "User ID that last updated this model group" + updatedBy: Int + "ID of the workflow that contains this model group" + workflowId: Int } -"An enumeration." -enum USERCHANGELOG_COLUMN_ENUM { - ID - DATE - USER_ID - USER_EMAIL - UPDATED_BY - UPDATER_EMAIL - PREVIOUSLY_ENABLED - ENABLED +"Component subclass for a Trainable Indico Model Group" +type ModelGroupComponent implements ComponentInterface { + "Actions that can be taken on this component" + actions: [Action] + "Workflow fields that this component can consume or produce" + availableFields: [WorkflowField] + "Class names output by this component" + classNames: [BaseTargetName] + componentAppId: Int + @deprecated( + reason: "This is no longer stored as a first-class property on the component model but in the config for certain types of components" + ) + componentStatus: ComponentStatus @deprecated(reason: "Not yet available") + componentType: ComponentType + "Names of outputs from upstream filters passed to this component" + filteredClasses: [String] + id: Int + "Model Group this component represents" + modelGroup: ModelGroup + "Model type of the model group this component represents" + modelType: ModelType + "Name of model group this component represents" + name: String + "Can the results of this component be used by our Review Interface?" + reviewable: Boolean + "Task type of the model group this component represents" + taskType: TaskType + "List of valid actions on the component" + validActions: [ValidAction] @deprecated(reason: "Use actions instead") } -type WorkflowPage { - workflows: [Workflow] +type ModelGroupPage { + modelGroups: [ModelGroup] pageInfo: PageInfo } -type Workflow { - id: Int - datasetId: Int - dataset: Dataset +type ModelLoad { + status: String +} + +type ModelOptions { + domain: FeatureDomainEnum + highQuality: Boolean + id: Int + interlabelerResolution: LabelResolutionStrategy + modelTrainingOptions: JSONString + predictOptions: JSONString + samplingStrategy: SamplingStrategy + seed: Int + testSplit: Float + weightByClassFrequency: Boolean + wordPredictorStrength: WordPredictorStrength +} + +type ModelPredict { + jobId: String +} + +type ModelSimilarity { + row: Int + value: Int +} + +type ModelStp { + autoReviewDenom: Int + autoReviewNumerator: Int + autoReviewStpPct: Float + modelGroupId: Int + name: String + reviewDenom: Int + reviewNumerator: Int + reviewStpPct: Float +} + +type ModelStpDailyFacts { + "Auto Review STP for model aggregated by parent filter context." + autoReviewStpForModel: Float + classStps: [ClassStpFacts] + modelGroupId: Int + name: String + "Review STP for model aggregated by parent filter context." + reviewStpForModel: Float + startDate: String + stps: [DailyStp] + workflowId: Int +} + +type ModelStpMetrics { + "Aggregate STP metrics for the model based on filters applied to the query" + aggregate: StpMetric + "STP metrics for each class associated with the model" + classMetrics: [ClassStpMetrics] + "Daily STP metrics for the model" + daily: [DailyStpMetric] + modelGroupId: Int + name: String +} + +type MultiClassLabel implements LabelInterface { + datasetuserId: Int + id: Int + label: _MultiClassLabel + rawLabel: [RawLabel] + taskType: TaskType +} + +type MultiClassPrediction implements PredictionInterface { + id: Int + modelId: Int + prediction: _MultiClassPred + rawPrediction: [RawPrediction] + taskType: TaskType +} + +type MultiTestResult { + actual: [String] + exampleId: Int + explanations: [Explanation] + predicted: [String] + rowIdx: Int @deprecated(reason: "Please use example_id") + score: [ClassConfidence] + text: String +} + +type Mutation { + activateUser(id: Int!): User + activeFormFields(files: [FileInput]): ActiveFormFields + "Start processing recently added CSV data files to the dataset" + addDataCsv( + "Ids of the DataFiles to process" + datafileIds: [Int] + "Id of the dataset to add the files to" + datasetId: Int! + ): Dataset + "Start processing recently added non-CSV data files to the dataset" + addDataFiles( + "Ids of the DataFiles to process" + datafileIds: [Int] + "Id of the dataset to add the files to" + datasetId: Int! + ): Dataset + addDataToWorkflow(workflowId: Int!): AddDataToWorkflow + """ + Add some uploaded files of any type to the dataset. We recommend + autoprocess=True, otherwise AddDataFiles or AddDataCSV should be + called after the files are added + """ + addDatasetFiles( + "Automatically process files that are uploaded and associated with the dataset" + autoprocess: Boolean = false + "Id of dataset to add files to" + datasetId: Int! + "A list of dicts representing each of the files to be uploaded. For each file, include `id` which must be some string, `name` which represents the name of the file you'd like to see in displays, `size` which is the size of the file, and `path` which is the /uploads/... Indico path the file currently resides at" + metadataList: JSONString! + ): Dataset + """ + Add user to dataset. If role permits, also add user to all tasks + The user must already be registered with the platform + """ + addDatasetUser( + "Id of the dataset to add the user to" + datasetId: Int! + "Email of the user to add to the dataset." + email: String! + "Dataset role the user should have" + role: Roles! + ): DatasetUser + "Add a new integration to a workflow. Currently supports exchange." + addExchangeIntegrationToWorkflow( + "Exchange configuration options" + config: ExchangeIntegrationConfigurationInput! + "Exchange credentials" + credentials: ExchangeIntegrationCredentialsInput! + "Id of the workflow to add the exchange integration to" + workflowId: Int! + ): AddExchangeIntegration + """ + Add a new field to an existing workflow. If component_id and target_name_ids arguments are additionally provided, this mutation also links the newly created field to the workflow component. + New field must have a name that is unique from any existing fields. + """ + addField( + "ID of the workflow component to link newly created field to" + componentId: Int + "The datatype of the field (e.g. 'date', 'price', etc.)" + datatype: String + "Configuration dictionary that controls how the normalized type is displayed in Indico Review UI" + formatConfig: GenericScalar + "Configuration dictionary that controls what inputs are recognized as valid instances of a given type" + inputConfig: GenericScalar + "Denotes whether a single field instance or multiple field instances are expected for an associated field. For instance, an invoice may have multiple 'Line Item Total's (multiple=True), but would be expected to only have a single 'Invoice Date' field (multiple=False)" + multiple: Boolean + "Field name displayed in review UI and the workflow result file" + name: String! + "Controls whether or not a valid instance of a field must be present to submit an review." + required: Boolean + "Target name IDs that the newly created field should be associated with. Must be target names associated with the provided component ID." + targetNameIds: [Int] + "Names of targets to associate with field (if any)." + targetNames: [String] + "Configuration that controls which additional validation checks should be run and what actions should be taken in case of their failure." + validationConfig: [ValidationInputConfig] + "Workflow ID" + workflowId: Int! + ): WorkflowField + "Add a class name to a labelset - Deprecated!" + addLabelsetTarget( + "Settings for creation of a new field to associate with the new target name." + fieldData: ComponentFieldInput + "Labelset to add the target name to" + labelsetId: Int! + "Name of the newly created target" + targetName: String! + "Labelset target name type" + targetType: TargetType + ): LabelSet @deprecated(reason: "Please use addModelGroupTargetNames") + addModelGroupComponent( + "Component ID to add the current component after (only provide when adding to a component)" + afterComponentId: Int + "Component link ID to add the current component after (only provide when adding to a link)" + afterLinkId: Int + "Use a blueprint to produce this component. Optional" + blueprintId: Int + "Dataset ID" + datasetId: Int! + "Label resolution strategy" + interlabelerResolution: LabelResolutionStrategy + "Labelset ID (use new_labelset_args to create a new labelset)" + labelsetColumnId: Int = null + "Model training options" + modelTrainingOptions: JSONString + "Model type" + modelType: ModelType + "Name of the model group" + name: String! + "Arguments for the new labelset to be created" + newLabelsetArgs: NewLabelsetInput + "Arguments for the new questionnaire to be created" + questionnaireArgs: QuestionnaireInput + "Workflow ID" + workflowId: Int! + ): AddModelGroupComponent + """ + Add more target names to a model group + Field data will be generated if not explicitly opted out of + """ + addModelGroupTargetNames( + "List of settings for creation of a new field/fields to associate with the new target name/names." + fieldData: [FieldInput] + "Id of the model group" + modelGroupId: Int! + "Target name information" + targetNames: [TargetNameInput]! + ): AddModelGroupTargetNames + addQuestionnaireUser(id: Int!, userId: Int!): QuestionnaireUser + addTarget(questionnaireId: Int!, target: String!): Question + """ + Add a component of any type to a workflow + 'component' can contain: + - component_type: ComponentType, required if not using blueprint + - name: String + - config: JSONString(dict) + """ + addWorkflowComponent( + afterComponentId: Int + afterComponentLinkId: Int + "Use a blueprint to produce this component." + blueprintId: Int + component: JSONString! + workflowId: Int! + ): AddWorkflowComponent + cancelModelTraining(modelId: Int!): CancelModelTraining + "Create a brand new, empty dataset" + createDataset( + "Configurations for a dataset or datacolumn" + config: DataConfigInput + "Type of data the dataset will hold. Defaults to DOCUMENT" + datasetType: DatasetType + "Name of the dataset to create" + name: String! + ): Dataset + createExport( + "Anonymize user information" + anonymous: Boolean = false + "Additional columns. Labelset's datacolumn is implicitly added" + columnIds: [Int] + "Combine labels from multiple labelers using this strategy" + combineLabels: LabelResolutionStrategy + datasetId: Int! + "Include datafile information" + fileInfo: Boolean = true + "Frozen labelsets of Labelset to limit examples by" + frozenLabelsetIds: [Int] + "Only include examples that have at least 1 label" + labeledOnly: Boolean = false + labelsetId: Int! + "Which models to include predictions from" + modelIds: [Int] + "Name for export" + name: String + ): Export + createModelGroup( + afterComponentId: Int + afterLinkId: Int + datasetId: Int! + domain: FeatureDomainEnum + finetune: Boolean + interlabelerResolution: LabelResolutionStrategy + labelsetColumnId: Int + makePredictions: Boolean = false + modelTrainingOptions: JSONString + modelType: ModelType + name: String! + rowIdx: [Int] + sourceColumnId: Int! + subsetId: Int + testSplit: Float = 0.2 + workflowId: Int! + ): ModelGroup @deprecated(reason: "Moved to AddModelGroupComponent") + createQuestionnaire( + "Enable predictions on the questionnaire" + activeLearning: Boolean = true + afterComponentId: Int + afterLinkId: Int + dataType: DataType! + datasetId: Int! + "Always use Text Labeling UI" + forceTextMode: Boolean + instructions: String + modelTrainingOptions: JSONString + modelType: ModelType + name: String! + numLabelersRequired: Int! + "Create a new questionnaire from an existing labelset." + originalLabelsetId: Int + questions: [QuestionInput]! + "Show predictions at the global level" + showPredictions: Boolean = true + sourceColumnId: Int! + "User IDs to add to the questionnaire" + users: [Int] + workflowId: Int! + ): Questionnaire @deprecated(reason: "Moved to AddModelGroupComponent") + createWorkflow(datasetId: Int!, name: String!): CreateWorkflow + createWorkflowFromBlueprint( + blueprintId: Int! + name: String + users: [UserInput] + ): CreateWorkflowFromBlueprint + deactivateUser(id: Int!): User + "Delete this dataset from the platform" + deleteDataset("Id of the dataset to delete" id: Int!): DeleteDataset + """ + Remove a file from a dataset. WARNING: do not use with non-FAILED files + which will result in non-deterministic behavior + """ + deleteDatasetFile( + "Id of dataset to remove the file from" + datasetId: Int! + "Id of the datafile to remove from the dataset" + fileId: Int! + ): Dataset + "Remove a user from a dataset. Also remove the user from all dataset tasks" + deleteDatasetUser( + "Id of the dataset to delete the user from" + datasetId: Int! + "Id of the platform user to modify permissions for" + userId: Int! + ): DeleteDatasetUser + deleteUserPreference(app: String!, key: String!): DeletePreference + deleteWorkflow(workflowId: Int!): DeleteWorkflow + deleteWorkflowComponent( + componentId: Int! + workflowId: Int! + ): DeleteWorkflowComponent + """ + Deletes an integration from a workflow. This permanently removes the integration. + You cannot undo this action. + """ + deleteWorkflowIntegration( + "The id of the integration to delete" + integrationId: Int! + ): DeleteIntegration + """ + Extract text from a document using one of our OCR engine providers. + Returns: List of ids of the document processing jobs kicked off by this mutation. Can be used to query for status and results. + """ + documentExtraction( + "List of file name and storage metadatas per document" + files: [FileInput] + """ + Preset or custom OCR configurations. + Full config information can be found here: + https://docs.indicodata.ai/articles/#!documentation-publication/ocr + """ + jsonConfig: JSONString + "Which OCR Engine to use" + ocrEngine: OCREngine = READAPI_V2 + ): DocumentExtraction + """ + Example: + mutation generate_new_refresh_token { + GenerateNewApiRefreshToken{ + refresh_token + } + } + """ + generateNewApiRefreshToken: GenerateNewApiRefreshToken + generateResetLink(userId: Int!): GenerateResetLink + invalidateSessions(id: Int!): User + modelLoad(modelId: Int!): ModelLoad + @deprecated(reason: "Models are loaded automatically on first predict") + modelPredict( + data: [String] + modelId: Int! + predictOptions: JSONString = null + ): ModelPredict + "Modify user's dataset role. If new role permits, also add user to all tasks" + modifyDatasetUser( + "Id of the dataset to modify the user for" + datasetId: Int! + "New dataset role the user should have" + role: Roles! + "Id of the platform user to modify permissions for" + userId: Int! + ): DatasetUser + modifyScopes(id: Int!, scopes: [Scope]!): User + "Create a new dataset with some files" + newDataset( + "DEPRECATED: Use kloudless uploader" + kloudless: Boolean = false + metadataList: JSONString! + ): Dataset @deprecated(reason: "Use createDataset instead") + "Given a field ID and a label of that field type, return a normalized version of the field." + normalizeLabel( + "Character end offset into the document" + end: Int + "Field ID" + fieldId: Int! + "Dictionary of any additional metadata associated with the text, e.g. model confidences: {'confidences': {...}}" + meta: GenericScalar + "Character start offset into the document" + start: Int + "Text to be normalized" + text: String! + ): [NormalizedLabel] + optimizeModelGroup(makePredictions: Boolean, modelGroupId: Int!): ModelGroup + """ + Pauses integration. Submissions will not be automatically processed + and new items will not be queued while paused. + """ + pauseWorkflowIntegration( + "The id of the integration to pause" + integrationId: Int! + ): PauseIntegration + pdfExtraction( + data: [String]! + images: Boolean + metadata: Boolean + pageFormat: String + rawText: Boolean + singleColumn: Boolean + tables: Boolean + text: Boolean + ): PDFExtraction + "Prune component field links and fields from deleted target names on a given workflow" + pruneFields("Workflow ID" workflowId: Int!): PruneFields + refreshViews( + "Force refresh views if the views were refreshed less than cooldown period ago." + force: Boolean = false + ): Refresh + removeQuestionnaireUser(id: Int!, userId: Int!): RemoveQuestionnaireUser + """ + Remove target names by id from a given workflow. Validate if target names can be + removed before removal. + """ + removeTargetNames( + "List of TargetName Ids to remove" + targetNameIds: [Int]! + "Workflow ID" + workflowId: Int! + ): RemoveTargetNames + requestStorageDownloadUrl( + "Storage URI with indico-file/indico-cache scheme" + uri: String! + ): RequestStorageDownloadUrl + requestStorageUploadUrl: RequestStorageUploadUrl + "Resume execution of a deferred task in a submission's workflow" + resumeSubmissionTask( + "Id of the component" + componentId: Int! + "Result to use for the resumed task" + response: ResumedTaskResponse! + "Id of the submission to resume" + submissionId: Int! + "Task UID to resume" + taskUid: String! + "Id of the running workflow" + workflowId: Int! + ): ResumeSubmissionTask + retrainModelGroup( + forceRetrain: Boolean + interlabelerResolution: LabelResolutionStrategy = null + modelGroupId: Int! + "Can only be updated on retrain for extraction models." + modelType: ModelType = null + workflowId: Int + ): ModelGroup + retrySubmissions(submissionIds: [Int]!): [Submission] + """ + Given a FAILED datafile on the dataset, provide an Indico or external URL + which will be downloaded and reprocessed for this datafile + """ + retryUrl( + "Id of the datafile to retry processing for" + datafileId: Int! + "Id of the dataset to retry a file for" + datasetId: Int! + "New external or internal Indico URL to process for the datafile" + newUrl: String! + ): Dataset + setUserExpiration( + "A date today or in the future to expire this user. Leave blank to remove user expiration date" + expirationDate: InputDate + id: Int! + ): User + "Starts integration. If integration is already running, this mutation is a no-op" + startWorkflowIntegration( + "The id of the integration to start" + integrationId: Int! + ): StartIntegration + submissionReport( + "Get all submissions, given valid permissions" + allSubmissions: Boolean = false + "Provide information about submissions, as they change over time" + changelog: Boolean = false + filters: SubmissionLogFilter + "Format of report to generate, defaults to CSV" + reportFormat: ReportFormat = CSV + ): GenerateSubmissionReport + submissionResults(submissionId: Int!): SubmissionResults + """ + Produce a Job which tracks the creation of a CSV containing all the flattened + results of the given submission ids + """ + submissionResultsReport( + "Ids of submissions to aggregate results for. Must provide btwn at least 1 and no more than 50 ids" + submissionIds: [Int!]! + "Workflow id to filter submissions from" + workflowId: Int! + ): Job + submitAutoReview( + changes: JSONString + "Bypass Review/Exception queue (not recommended)" + forceComplete: Boolean = false + rejected: Boolean = false + submissionId: Int! + ): SubmitAutoReview + "Deprecated - assumes all row_indices reflect example id" + submitLabels( + datasetId: Int! + labels: [SubmissionLabel]! + labelsetId: Int! + "Model group to retrain after label submission" + modelGroupId: Int + ): SubmitLabels @deprecated(reason: "Please use SubmitLabelsV2") + submitLabelsV2( + labels: [LabelInput]! + labelsetId: Int! + "Model group to retrain after label submission" + modelGroupId: Int + ): SubmitLabelsV2 + submitReview( + changes: JSONString + notes: String + rejected: Boolean = false + submissionId: Int! + ): Review + toggleWorkflowAutoReview( + "All new submissions will wait for Auto Review" + enableAutoReview: Boolean! + "If toggling auto review on, mark existing subs pending review as pending auto review. Ignore if toggling off" + updateExistingSubmissions: Boolean = false + workflowId: Int! + ): Workflow + @deprecated( + reason: "Replaced by UpdateWorkflowMeta, toggling on workflow settings update" + ) + toggleWorkflowReview( + "If toggling review off, mark existing submissions waiting for review as complete. Ignored if toggling review on." + completeExistingSubmissions: Boolean = false + "Place all future submissions into review queue" + enableReview: Boolean! + workflowId: Int! + ): Workflow + @deprecated( + reason: "Replaced by UpdateWorkflowMeta, toggling on workflow settings update" + ) + unlockUser(id: Int!): User + "Update properties of the dataset" + updateDataset( + datasetId: Int! + "New name of the dataset." + name: String + ): Dataset + "Update the workflow field with the provided ID." + updateField( + "The datatype of the field (e.g. 'date', 'price', etc.)" + datatype: String + fieldId: Int! + "Configuration dictionary that controls how the normalized type is displayed in Indico Review UI" + formatConfig: GenericScalar + "Configuration dictionary that controls what inputs are recognized as valid instances of a given type" + inputConfig: GenericScalar + "Denotes whether a single field instance or multiple field instances are expected for an associated field. For instance, an invoice may have multiple 'Line Item Total's (multiple=True), but would be expected to only have a single 'Invoice Date' field (multiple=False)" + multiple: Boolean + "Field name displayed in review UI and the workflow result file" + name: String + "Controls whether or not a valid instance of a field must be present to submit an review." + required: Boolean + "Configuration that controls which additional validation checks should be run and what actions should be taken in case of their failure." + validationConfig: [ValidationInputConfig] + ): WorkflowField + updateLabelset( + labelsetId: Int! + "Minimum number of labelers required to label each example" + numLabelersRequired: Int + ): LabelSet + updateLabelsetTargetPositions( + labelsetId: Int! + targetNames: [String] + ): LabelSet + updateModelGroupName(modelGroupId: Int!, name: String!): ModelGroup + updateModelGroupSettings( + domain: FeatureDomainEnum + finetune: Boolean + interlabelerResolution: LabelResolutionStrategy + makePredictions: Boolean + modelGroupId: Int! + modelTrainingOptions: JSONString + predictOptions: JSONString + rocAucAveraging: RocAucAveraging + samplingStrategy: SamplingStrategy + taskType: TaskType + testSplit: Float + wordPredictorStrength: WordPredictorStrength + ): ModelGroup + updateQuestionKeywords( + "Use keywords for all users" + globalPreference: Boolean = false + keywords: [String]! + questionnaireId: Int! + ): UpdateKeywords + updateQuestionnaire( + "Enable labeling tasks" + active: Boolean + "Enable predictions on the questionnaire" + activeLearning: Boolean + dataType: DataType + id: Int! + instructions: String + name: String + "Show predictions at the global level" + showPredictions: Boolean + ): Questionnaire + updateSubmission( + "Mark the submission as having been retrieved" + retrieved: Boolean + submissionId: Int! + ): Submission + updateUser(name: String): User + updateUserPreference( + app: String! + key: String! + value: JSONString! + ): Preference + updateWorkflowMeta( + "Estimated human time to complete the workflow in minutes" + estHumanTimeMins: Int + name: String + settings: ReviewSettingsInput + workflowId: Int! + ): Workflow + userChangelogReport( + "Changelog up to this date (23:59 UTC)" + endDate: Date + filters: UserReportFilter + "Format of report to generate, defaults to CSV" + reportFormat: ReportFormat + "Changelog from this date (00:00 UTC)" + startDate: Date + ): GenerateUserChangelogReport + userSnapshotReport( + "User information on this date (23:59 UTC)" + date: Date + filters: UserReportFilter + "Format of report to generate, defaults to CSV" + reportFormat: ReportFormat + ): GenerateUserSnapshotReport + """ + Validates credentials and configuration for Exchange Integration. Does not create an integration. + Credentials are validated before the config. Correct credentials must be passed in before config is validated. + """ + validateExchangeConfigCredentials( + "Exchange configuration options" + config: ExchangeIntegrationConfigurationInput! + "Exchange credentials" + credentials: ExchangeIntegrationCredentialsInput! + ): ValidateExchangeConfigCredentials + """ + Submit files to a workflow for processing + Returns: SubmissionResult object which contains ids that can be queried for status and results. + """ + workflowSubmission( + "Batch all files under a single submission" + bundle: Boolean = false + "UUID for duplicate Submissions caching" + duplicationId: String + "List of FileInput objects" + files: [FileInput]! + "Submission output result file version" + resultVersion: SubmissionResultVersion + "Id of the workflow to submit to" + workflowId: Int! + ): SubmissionResult + workflowUrlSubmission( + "Batch all urls under a single submission" + bundle: Boolean = false + "UUID for duplicate Submissions caching" + duplicationId: String + "Submission output result file version" + resultVersion: SubmissionResultVersion = null + urls: [String]! + workflowId: Int! + ): SubmissionResult +} + +"The result of normalizing and running all validation rules on a provided text extraction." +type NormalizedLabel { + "End offset of extraction in the document (chars)" + end: Int + "A formatted version of the extracted structured data (if normalization succeeded). Same as `text` if normalization failed." + formatted: String + "Copy of any metadata passed to the normalization mutation (e.g. confidence info)" + meta: GenericScalar + "Start offset of extraction in the document (chars)" + start: Int + "Overall status. If any validation errors occurred, equates to the highest severity error." + status: ValidationStatus + "A structured representation of the data (if normalization succeeded). None if normalization failed." + structured: GenericScalar + "Original extraction text" + text: String + "Results of all individual validation rules" + validation: [ValidationResult] +} + +type OBJECT_DETECTIONTestResult { + falseNegative: [MultiTestResult] + falsePositive: [MultiTestResult] + modelId: Int + resultCounts: ResultCounts + threshold: Float + trueNegative: [MultiTestResult] + truePositive: [MultiTestResult] +} + +""" +The options for OCR Engines available +on this platform +""" +type OCREngineOptions { + "The default OCR engine that will be used for datasets" + defaultEngine: OCREngine + "All the options for each engine available on this platform" + engines: [EngineOptions] + "The engines available on this platform" + types: [OCREngineType] +} + +"Represents a type of OCR Engine on the platform" +type OCREngineType { + description: String + name: String +} + +"Language that can be parsed by the OCR Engine" +type OCRLanguage { + code: String + name: String +} + +type ObjectDetectionEvaluation { + metrics: JSONString + testResults(actual: [String], threshold: Float): OBJECT_DETECTIONTestResult +} + +type OmnipageOcrOptions implements OCROptions { + "Auto rotate" + autoRotate: Boolean + "Return table information for post-processing rules" + cells: Boolean + "Force render" + forceRender: Boolean + "List of languages to use" + languages: [omnipageLanguageCode] + "Native layout" + nativeLayout: Boolean + "Native PDF" + nativePdf: Boolean + ocrEngine: OCREngine + ocrEngineVersion: String + "Read table as a single column" + singleColumn: Boolean + "PDF split version" + splitVersion: Int + "Spreadsheet converter version (supported versions: 1,2,3,4)" + spreadsheetConverterVersion: Int + "Read table in row or column order" + tableReadOrder: TableReadOrder + "Scale up low resolution images" + upscaleImages: Boolean +} + +type PDFExtraction { + jobId: String +} + +type PRCurve { + confidences: [String] + name: String + precision: [Float] + recall: [Float] +} + +""" +Representation for paginated results for a List query +Use `endCursor` as part of the `after` parameter to get the next page +or `startCursor` as part of the `before` parameter to get to the previous page +""" +type PageInfo { + "Total number of results for this query" + aggregateCount: Int + "Cursor on the last result - from which the next page can be acquired" + endCursor: Int + "Another page is available" + hasNextPage: Boolean + "Cursor on the first result - from which the previous page can be acquired" + startCursor: Int +} + +""" +Pauses integration. Submissions will not be automatically processed +and new items will not be queued while paused. +""" +type PauseIntegration { + "True if the integration is succesfully paused" + success: Boolean +} + +type PredictionMetric { + numPreds: Int +} + +type PredictionMetrics { + "Total number of model generated predictions for this workflow [PREDICTIONS_COUNT]" + aggregate: PredictionMetric + "Number of model generated predictions for the workflow, each day [PREDICTIONS_COUNT]" + daily: [DailyPredictionMetric] +} + +type Preference { + app: String + key: String + value: JSONString +} + +type Processor { + args: JSONString + processorType: ProcessorType! +} + +"Prune component field links and fields from deleted target names on a given workflow" +type PruneFields { + "Success" + success: Boolean +} + +type Question { + DatasetId: Int + SubsetId: Int + id: Int + keywords: [String] + labelset: LabelSet + labelsetId: Int + modelGroup: ModelGroup + modelGroupId: Int + questionnaireId: Int + status: QuestionStatus + "Ordered list of target names." + targets: [String] @deprecated(reason: "Use labelset.targetNames") + text: String + type: TaskType +} + +"Indico Teach Task - has some settings and 1 Question" +type Questionnaire { + "Labeling tasks are enabled" + active: Boolean + "Predictions are enabled" + activeLearning: Boolean + assignedUsers: [QuestionnaireUser] + "Unix timestamp" createdAt: String + "User id" createdBy: Int + dataType: DataType + datasetId: Int + "Examples which still require labeling" + examples(datafileId: Int, numExamples: Int!): [Example] + "Labeling will always be done in Text mode" + forceTextMode: Boolean + "Unique ID of the questionnaire" + id: Int + instructions: String + labelsetId: Int + modelGroupId: Int + name: String + numFullyLabeled: Int + numLabeledByMe: Int + numRejected: Int + numTotalExamples: Int + "On-document labeling interface enabled" + odl: Boolean + processors: [Processor] + question: Question + questions: [Question] @deprecated(reason: "Use question") + "cumulative status of all questions in questionnaire" + questionsStatus: QuestionStatus @deprecated(reason: "Use status") + role: Roles + "Show predictions at the global level" + showPredictions: Boolean + sourceColumnId: Int + status: QuestionStatus + subsetId: Int + "Unix timestamp" updatedAt: String + "User id" updatedBy: Int - name: String - docbots: [DocBot] - userRole: Roles - status: WorkflowStatus - reviewable: Boolean - reviewEnabled: Boolean - autoReviewEnabled: Boolean - "Estimated human time to complete the workflow in minutes" - estHumanTimeMins: Int - submissionCounts: SubmissionCounts - reviewableModelGroups: [ModelGroup] - submissionFacts: SubmissionFacts - metrics(startDate: Date endDate: Date): WorkflowMetrics } -type Dataset { - id: Int - name: String - status: DatasetStatus - rowCount: Int - defaultSubsetId: Int - type: DatasetType - errorInfo: String +type QuestionnairePage { + pageInfo: PageInfo + questionnaires: [Questionnaire] +} + +type QuestionnaireUser { + "Time the questionnaire user was created" createdAt: String + "User ID of the user that created the questionnaire" createdBy: Int - updatedAt: String - updatedBy: Int - datacolumns: [DataColumn] - labelsets: [LabelSet] + datasetId: Int + "Email of the user associated with the questionnaire" + email: String + id: Int + labelCount: Int + "Name of the user associated with the questionnaire" + name: String permissions: [String] - exports: [Export] - modelGroups(taskTypes: [TaskType]): [ModelGroup] - modelGroup(id: Int): ModelGroup - files: [DataFile] - numFireGroups: Int - numModelGroups: Int - numQuestionnaires: Int - dataInfo(labelsetId: Int sourceColumnId: Int): DataInfo @deprecated(reason: "Fetch class balance info under ModelGroup") - users: [DatasetUser] + questionnaireId: Int + role: Roles + "ID of the user associated with the questionnaire" + userId: Int } -"An enumeration." -enum DatasetStatus { - STAGED - PROCESSED - UPLOADING - CREATING - FAILED - COMPLETE - DELETING +type QueueMetrics { + dailyCumulative: [DailyQueueMetric] } -"An enumeration." -enum DatasetType { - IMAGE - TEXT - DOCUMENT +type RATIONALIZED_CLASSIFICATIONTestResult { + falseNegative: [SingleTestResult] + falsePositive: [SingleTestResult] + modelId: Int + resultCounts: ResultCounts + threshold: Float + trueNegative: [SingleTestResult] + truePositive: [SingleTestResult] } -type DataColumn { - id: Int +type ROCCurve { + auc: Float + confidences: [String] + falsePositiveRate: [Float] name: String - datasetId: Int - columnIndex: Int - datatype: DataTypes - ocrUsed: Boolean + truePositiveRate: [Float] } -"An enumeration." -enum DataTypes { - CATEGORICAL - STRING - NUMERIC - IMAGE - UNKNOWN +type RationalizedClassificationEvaluation { + confusionResult(actual: String!, predicted: String!): [SingleTestResult] + metrics: ClassificationModelMetrics + testResults( + actual: String + threshold: Float + ): RATIONALIZED_CLASSIFICATIONTestResult } -type LabelSet { - id: Int - name: String - numLabelersRequired: Int - numLabelsetPoints: Int - numFullyLabeled: Int - taskType: TaskType - targetNames: [TargetName] +type RationalizedClassificationPrediction { + confidences: [ClassConfidence] + explanations: [Explanation] + label: String + tokenPredictions: [TokenPrediction] } -"An enumeration." -enum TaskType { - CLASSIFICATION - CLASSIFICATION_MULTIPLE - RATIONALIZED_CLASSIFICATION - REGRESSION - ANNOTATION - OBJECT_DETECTION - FORM_EXTRACTION +type RawLabel { + "pixel bounds of the image" + bounds: [SpatialSpan] + "Class name selected for this label" + clsName: TargetName + "character bounds of the text" + spans: [TokenSpan] +} + +type RawPrediction { + "pixel bounds of the image" + bounds: [SpatialSpan] + "Confidences of all predicted classes" + classConfidences: [ClassConfidenceV2] + "Chosen class predicted by model" + clsName: TargetName + "character bounds of the text" + spans: [TokenSpan] +} + +type ReadapiOcrOptions implements OCROptions { + "Auto rotate" + autoRotate: Boolean + "List of languages to use" + languages: [readapiLanguageCode] + ocrEngine: OCREngine + ocrEngineVersion: String + "Read table as a single column" + singleColumn: Boolean + "Spreadsheet converter version (supported versions: 1,2,3,4)" + spreadsheetConverterVersion: Int + "Scale up low resolution images" + upscaleImages: Boolean +} + +type ReadapiTablesV1OcrOptions implements OCROptions { + "Auto rotate" + autoRotate: Boolean + "List of languages to use" + languages: [readapiTablesV1LanguageCode] + ocrEngine: OCREngine + ocrEngineVersion: String + "Read table as a single column" + singleColumn: Boolean + "Spreadsheet converter version (supported versions: 1,2,3,4)" + spreadsheetConverterVersion: Int + "Scale up low resolution images" + upscaleImages: Boolean +} + +type ReadapiV2OcrOptions implements OCROptions { + "Auto rotate" + autoRotate: Boolean + "List of languages to use" + languages: [readapiV2LanguageCode] + ocrEngine: OCREngine + ocrEngineVersion: String + "Read table as a single column" + singleColumn: Boolean + "Spreadsheet converter version (supported versions: 1,2,3,4)" + spreadsheetConverterVersion: Int + "Scale up low resolution images" + upscaleImages: Boolean } -type TargetName { - id: Int - labelsetId: Int - name: String - position: Int +type Refresh { + refreshStartedAt: DateTimeISO + refreshedAt: DateTimeISO + refreshing: Boolean } -type Export { +type RefreshTokenMeta { + createdAt: DateTime id: Int - datasetId: Int - name: String - status: ExportStatus - columnIds: [Int] - labelsetIds: [Int] - subsetIds: [Int] - numLabels: Int - anonymous: Boolean - "Download URL of this export" - downloadUrl: String - "Unix timestamp" - createdAt: String - "user id" - createdBy: Int + isApiToken: Boolean + isValid: Boolean + userId: Int } -"An enumeration." -enum ExportStatus { - STARTED - FAILED - COMPLETE +type RemoveQuestionnaireUser { + success: Boolean } -type ModelGroup { - id: Int - datasetId: Int - name: String - status: ModelStatus - models: [Model] - model(id: Int): Model - selectedModel: Model - taskType: TaskType - dataType: DataTypes - retrainRequired: Boolean - labelset: LabelSet - sourceColumn: DataColumn - interlabelerResolution: LabelResolutionStrategy - example(rowIndex: Int): Example - createdAt: String - createdBy: Int - updatedAt: String - updatedBy: Int - "Ordered list of target names used to train the model." - classNames: [String] - processors: [Processor] - subsetId: Int - dataInfo: DataInfo - labelsetColumnId: Int +""" +Remove target names by id from a given workflow. Validate if target names can be +removed before removal. +""" +type RemoveTargetNames { + "Success" + success: Boolean + "List of TargetName Ids removed" + targetNameIds: [Int] } -"An enumeration." -enum ModelStatus { - CREATING - TRAINING - FAILED - COMPLETE - NOT_ENOUGH_DATA +type RequestStorageDownloadUrl { + "Relative path" + relativePath: String + "Signed URL" + signedUrl: String } -type Model { - id: Int - collectionName: String - predictionLabelsetId: Int - classNames: [String] - evaluation: Evaluation - predictions(sources: [String] threshold: Float): [Prediction] - modelInfo: JSONString - createdAt: String - createdBy: Int - updatedAt: String - updatedBy: Int - status: ModelStatus - trainingSubsetId: Int - testingSubsetId: Int - trainingProgress: TrainingProgress - linkable: Boolean - modelType: ModelType - rareClasses: [ClassCount] - unlabeledClasses: [String] +type RequestStorageUploadUrl { + "Relative path" + relativePath: String + "Signed URL" + signedUrl: String } -union Evaluation = ClassificationSingleEvaluation | ClassificationMultipleEvaluation | AnnotationEvaluation | ObjectDetectionEvaluation | RationalizedClassificationEvaluation - -type ClassificationSingleEvaluation { - metrics: ClassificationModelMetrics - confusionResult(predicted: String! actual: String!): [SingleTestResult] - testResults(actual: String threshold: Float): CLASSIFICATIONTestResult +type ResultCounts { + falseNegative: Float + falsePositive: Float + trueNegative: Float + truePositive: Float } -type ClassificationModelMetrics { - metricsTable: [MetricsTable] - rocCurves: [ROCCurve] - prCurves: [PRCurve] - confusionMatrix: ConfusionMatrix +"Resume execution of a deferred task in a submission's workflow" +type ResumeSubmissionTask { + success: Boolean } -type MetricsTable { - name: String - auc: Float - accuracy: Float - precision: Float - recall: Float - f1Score: Float +"Represents a review for a submission created by a machine (AUTO_REVIEW) or reviewer" +type Review { + adminReview: Boolean @deprecated(reason: "Please use review_type") + "Changes for this review. This field is not generally available. Use submission result files instead" + changes: JSONString + "When this review was completed by the reviewer" + completedAt: String + "When this user first opened the file. See started_at as well" + createdAt: String + "Reviewer id" + createdBy: Int + id: Int + notes: String + rejected: Boolean + reviewType: ReviewType + "When this review was started. Differs from created_at because a reviewer may restart their review at any time" + startedAt: String + submissionId: Int } -type ROCCurve { - confidences: [String] +"Subclass for a component representing the Review stage(s) in a workflow" +type ReviewComponent implements ComponentInterface { + "Actions that can be taken on this component" + actions: [Action] + "Enable the Auto review queue" + autoReviewQueueEnabled: Boolean + "Workflow fields that this component can consume or produce" + availableFields: [WorkflowField] + componentAppId: Int + @deprecated( + reason: "This is no longer stored as a first-class property on the component model but in the config for certain types of components" + ) + componentStatus: ComponentStatus @deprecated(reason: "Not yet available") + componentType: ComponentType + "Add value to Exceptions queue submissions in the workflow" + exceptionsQueueAddValueEnabled: Boolean + "Enable the Exceptions queue" + exceptionsQueueEnabled: Boolean + "Add a rejection reason to review queue submissions in the workflow" + exceptionsQueueRejectionReasonRequired: Boolean + "Enable Submissions List for exceptions queue. Accessible from the Review List page." + exceptionsSubmissionListEnabled: Boolean + "Names of outputs from upstream filters passed to this component" + filteredClasses: [String] + id: Int + "Name of component" name: String - truePositiveRate: [Float] - falsePositiveRate: [Float] - auc: Float + "Add value to Review queue submissions in the workflow" + reviewQueueAddValueEnabled: Boolean + "Enable the Review queue. If disabled, also disables the Exceptions and Auto review queues" + reviewQueueEnabled: Boolean + "Add a rejection reason to Review queue submissions in the workflow" + reviewQueueRejectionReasonRequired: Boolean + "Required number of reviewers per submission in the workflow" + reviewQueueReviewersRequired: Int + "Enable Submissions List for review queue. Accessible from the Review List page." + reviewSubmissionListEnabled: Boolean + "Can the results of this component be used by our Review Interface?" + reviewable: Boolean + "List of valid actions on the component" + validActions: [ValidAction] @deprecated(reason: "Use actions instead") +} + +type ReviewSettings { + "Enable the Auto review queue" + autoReviewQueueEnabled: Boolean + "Add value to Exceptions queue submissions in the workflow" + exceptionsQueueAddValueEnabled: Boolean + "Enable the Exceptions queue" + exceptionsQueueEnabled: Boolean + "Add a rejection reason to review queue submissions in the workflow" + exceptionsQueueRejectionReasonRequired: Boolean + "Enable Submissions List for exceptions queue. Accessible from the Review List page." + exceptionsSubmissionListEnabled: Boolean + "Add value to Review queue submissions in the workflow" + reviewQueueAddValueEnabled: Boolean + "Enable the Review queue. If disabled, also disables the Exceptions and Auto review queues" + reviewQueueEnabled: Boolean + "Add a rejection reason to Review queue submissions in the workflow" + reviewQueueRejectionReasonRequired: Boolean + "Required number of reviewers per submission in the workflow" + reviewQueueReviewersRequired: Int + "Enable Submissions List for review queue. Accessible from the Review List page." + reviewSubmissionListEnabled: Boolean +} + +type Schema { + allUsers( + "Find results after this cursor" + after: Int + "Find results before this cursor" + before: Int + "Return results in descending order" + desc: Boolean + filters: UserFilter + "Include scopes for each user" + includeScopes: Boolean + "Max number of results to return" + limit: Int + "attribute to order results by" + orderBy: USER_COLUMN_ENUM + "number of pages to skip" + skip: Int + ): UserPage + checkoutSpecificSubmission(submissionId: Int!): Submission + "Retrieve a datafile" + datafile(datafileId: Int!): DataFile + "Retrieve multiple datafiles" + datafiles(datafileIds: [Int]!): [DataFile] + "Get a single dataset by id" + dataset(id: Int): Dataset + datasets(permissions: [PermissionType]): [Dataset] + datasetsPage( + "Find results after this cursor" + after: Int + "Find results before this cursor" + before: Int + "Return results in descending order" + desc: Boolean + filters: DatasetFilter + "Max number of results to return" + limit: Int + "attribute to order results by" + orderBy: DATASET_COLUMN_ENUM + permissions: [PermissionType] + showAll: Boolean + "number of pages to skip" + skip: Int + ): DatasetPage + exports( + "Find results after this cursor" + after: Int + "Find results before this cursor" + before: Int + columnIds: [Int] + datasetId: Int + "Return results in descending order" + desc: Boolean + exportIds: [Int] + frozenLabelsetIds: [Int] + labelsetIds: [Int] + "Max number of results to return" + limit: Int + "attribute to order results by" + orderBy: EXPORT_COLUMN_ENUM + "number of pages to skip" + skip: Int + ): ExportPage + field("Field ID" fieldId: Int): WorkflowField + fieldTypes( + name: String + "If provided, return only fields compatible with this task type." + taskType: TaskType + workflowId: Int + ): [WorkflowFieldType] + fields( + "Name of a specific field on the specified workflow." + name: String + "Workflow ID" + workflowId: Int! + ): [WorkflowField] + "Get a single model by model file path" + findModel("File path of the model to find" modelFilePath: String!): Model + gallery: Gallery + ipaVersion: String + job(id: String): Job + modelGroup(modelGroupId: Int!): ModelGroup + modelGroups( + "Find results after this cursor" + after: Int + "Find results before this cursor" + before: Int + datasetIds: [Int] + "Return results in descending order" + desc: Boolean + filters: ModelGroupFilter + "Max number of results to return" + limit: Int + modelGroupIds: [Int] + "attribute to order results by" + orderBy: MODELGROUP_COLUMN_ENUM + "number of pages to skip" + skip: Int + ): ModelGroupPage + modelSimilarity( + modelGroupId: Int! + modelId: Int! + query: String! + ): [ModelSimilarity] + ocrOptions: OCREngineOptions + oneUser(id: Int!): User + questionnaire(id: Int!): Questionnaire + questionnaires( + "Find results after this cursor" + after: Int + "Find results before this cursor" + before: Int + datasetIds: [Int] + "Return results in descending order" + desc: Boolean + filters: QuestionnaireFilter + "Max number of results to return" + limit: Int + "attribute to order results by" + orderBy: QUESTIONNAIRE_COLUMN_ENUM + questionnaireIds: [Int] + "number of pages to skip" + skip: Int + ): QuestionnairePage + randomSubmission(adminReview: Boolean = false, workflowId: Int!): Submission + refresh: Refresh + "Search for some piece of text within a particular datapoint on the dataset" + searchDatapoint( + "Max result length including keyword and surrounding text" + context: Int + datapointId: Int! + "Use case-insensitive search" + ignoreCase: Boolean = true + "Keyword to search the text for" + keyword: String! + "Use a regex keyword to find matches in the text" + regex: Boolean + ): [TextSearchResult] @deprecated(reason: "Use search example instead") + submission(id: Int!): Submission + submissions( + "Find results after this cursor" + after: Int + "Find results before this cursor" + before: Int + "Return results in descending order" + desc: Boolean + filters: SubmissionFilter + "Max number of results to return" + limit: Int + "attribute to order results by" + orderBy: SUBMISSION_COLUMN_ENUM + "number of pages to skip" + skip: Int + submissionIds: [Int] + workflowIds: [Int] + ): SubmissionPage + submissionsLog( + "Find results after this cursor" + after: Int + "Get all submissions, given valid permissions" + allSubmissions: Boolean = false + "Find results before this cursor" + before: Int + "Include info about submissions as they change over time" + changelog: Boolean = false + "Return results in descending order" + desc: Boolean + filters: SubmissionLogFilter + "Max number of results to return" + limit: Int + "attribute to order results by" + orderBy: SUBMISSIONEVENT_COLUMN_ENUM + "number of pages to skip" + skip: Int + ): SubmissionEventPage + user: User + userChangelog( + "Find results after this cursor" + after: Int + "Find results before this cursor" + before: Int + "Return results in descending order" + desc: Boolean + "Get changes on or before this day (23:59 UCT)" + endDate: Date + filters: UserReportFilter + "Max number of results to return" + limit: Int = 100 + "attribute to order results by" + orderBy: USERCHANGELOG_COLUMN_ENUM + "number of pages to skip" + skip: Int + "Get changes on or after this daye (00:00 UTC)" + startDate: Date + ): UserChangelogPage + userSnapshot( + "Find results after this cursor" + after: Int + "Find results before this cursor" + before: Int + "Snapshot of permissions at this date (23:59 UTC)" + date: Date + "Return results in descending order" + desc: Boolean + filters: UserReportFilter + "Max number of results to return" + limit: Int = 100 + "attribute to order results by" + orderBy: USERSNAPSHOT_COLUMN_ENUM + "number of pages to skip" + skip: Int + ): UserSnapshotPage + userSummary("User summary at this date (23:59 UTC)" date: Date): UserSummary + workflowBlueprints( + "Find results after this cursor" + after: Int + "Find results before this cursor" + before: Int + "Return results in descending order" + desc: Boolean + filters: WorkflowBlueprintFilter + "Max number of results to return" + limit: Int + "attribute to order results by" + orderBy: WORKFLOWBLUEPRINT_COLUMN_ENUM + "number of pages to skip" + skip: Int + ): WorkflowBlueprintPage + @deprecated(reason: "Use gallery.workflow.blueprintsPage instead") + workflows( + "Find results after this cursor" + after: Int + "Find results before this cursor" + before: Int + datasetIds: [Int] + "Return results in descending order" + desc: Boolean + filters: WorkflowFilter + "Max number of results to return" + limit: Int + metricsStartDate: Date + "attribute to order results by" + orderBy: WORKFLOW_COLUMN_ENUM + role: Roles + "number of pages to skip" + skip: Int + workflowIds: [Int] + ): WorkflowPage } -type PRCurve { - confidences: [String] - name: String - precision: [Float] - recall: [Float] +type ScopeAccess { + scope: Scope + userId: Int } -type ConfusionMatrix { - classes: [String] - matrix: [[Float]] +"The latest representation of a label" +type SimpleLabel implements LabelInterface { + datasetuserId: Int + id: Int + rawLabel: [RawLabel] + taskType: TaskType +} + +"The latest representation of a prediction" +type SimplePrediction implements PredictionInterface { + id: Int + modelId: Int + rawPrediction: [RawPrediction] + taskType: TaskType } type SingleTestResult { - explanations: [Explanation] - text: String - rowIdx: Int actual: String + exampleId: Int + explanations: [Explanation] predicted: String + rowIdx: Int @deprecated(reason: "Please use example_id") score: [ClassConfidence] -} - -type Explanation { - label: String - metadata: ExplainMeta - similarity: Float text: String } -type ExplainMeta { - datasetId: Int - rowIndex: Int - rowId: Int - sourceColumnId: Int -} - -type ClassConfidence { - name: String - confidence: Float -} - -type CLASSIFICATIONTestResult { - falsePositive: [SingleTestResult] - falseNegative: [SingleTestResult] - truePositive: [SingleTestResult] - trueNegative: [SingleTestResult] - resultCounts: ResultCounts - threshold: Float - modelId: Int +type SpatialSpan { + bottom: Int + ctxId: Int + idx: Int + left: Int + pageNum: Int + right: Int + top: Int + "image url" + value: String } -type ResultCounts { - falsePositive: Float - falseNegative: Float - truePositive: Float - trueNegative: Float +"Starts integration. If integration is already running, this mutation is a no-op" +type StartIntegration { + "True if the integration is succesfully started" + success: Boolean } -type ClassificationMultipleEvaluation { - metrics: ClassificationModelMetrics - confusionResult(predicted: String! actual: String!): [SingleTestResult] - testResults(actual: [String] threshold: Float): CLASSIFICATION_MULTIPLETestResult +type StpFacts { + model: [ModelStp] } -type CLASSIFICATION_MULTIPLETestResult { - falsePositive: [MultiTestResult] - falseNegative: [MultiTestResult] - truePositive: [MultiTestResult] - trueNegative: [MultiTestResult] - resultCounts: ResultCounts - threshold: Float - modelId: Int +type StpFactsDaily { + model: [ModelStpDailyFacts] + startDate: String + workflow: [DailyStp] + workflowId: Int } -type MultiTestResult { - explanations: [Explanation] - text: String - rowIdx: Int - actual: [String] - predicted: [String] - score: [ClassConfidence] +type StpMetric { + "The union of user supplied labels and auto review labels [PREDICTIONS_STP]" + autoReviewDenom: Int + "The number of human accepted auto review labels [PREDICTIONS_STP]" + autoReviewNumerator: Int + "Auto review numerator divided by auto review denomoinator, applicable if auto-review is enabled [PREDICTIONS_STP]" + autoReviewStpPct: Float + "The union of user supplied labels and model predictions [PREDICTIONS_STP]" + reviewDenom: Int + "The number of human accepted model predictions that exactly match model predictions [PREDICTIONS_STP]" + reviewNumerator: Int + "Review numerator divided by review denominator, applicable if review is enabled and auto-review is disabled [PREDICTIONS_STP]" + reviewStpPct: Float } -type AnnotationEvaluation { - metrics: AnnotationModelMetrics - examples("filter on a specific label" label: String "filter source data on a string" textSearch: String "number of pages to skip" skip: Int "Find results before this cursor" before: Int "Find results after this cursor" after: Int "Max number of results to return" limit: Int "Return results in descending order" desc: Boolean "attribute to order results by" orderBy: ANNOTATIONEXAMPLE_COLUMN_ENUM includeAggregate: Boolean = false): AnnotationExamplesPage - testResults(actual: [String] threshold: Float): ANNOTATIONTestResult +type StpMetrics { + "Schema for model STP metrics including class STP metrics as child nodes on this object's schema" + model: [ModelStpMetrics] + "STP metrics aggregate at the level of the workflow" + workflow: WorkflowStpMetrics } -type AnnotationModelMetrics { - "Metrics for evaluating model performance per class" - classMetrics: [AnnotationClassMetrics] - "Metrics for evaluating model performance at the model level, across classes" - modelLevelMetrics: [ModelLevelMetrics] - "Model retraining is required in order to calculate metrics." - retrainForMetrics: Boolean +"Object containing data associated with a workflow submission" +type Submission { + "Internal field for review load" + AutoReviewLoaded: Boolean + "Latest auto review for submission" + autoReview: Review + "Datetime the submission reached a completed state" + completedAt: DateTime + "Datetime the submission was created" + createdAt: DateTime + "ID of the user who created the submission" + createdBy: Int + "ID of the dataset associated with the submission" + datasetId: Int + "DEPRECATED: Submission files have been deleted from file store" + deleted: Boolean @deprecated(reason: "Now uses `files_deleted`") + "Errors occurred during this submission" + errors: String + "Submission files have been deleted from file store" + filesDeleted: Boolean + "Unique ID of the submission" + id: Int + "Local URL to first stored input" + inputFile: String + "Original name of first file" + inputFilename: String + inputFiles: [SubmissionFile] + "OCR engine used for submission" + ocrEngine: String + outputFiles: [SubmissionOutput] + "Local URL to most recently stored output" + resultFile: String + retries: [SubmissionRetry] + "Submission has been marked as having been retrieved" + retrieved: Boolean + "True if the submission is being actively reviewed" + reviewInProgress: Boolean + "completed reviews of this submission, without changes" + reviews( + "include pending and incomplete reviews in this list" + allReviews: Boolean = false + ): [Review] + "List of TextSearchResults" + searchResult( + "Max result length including keyword and surrounding text" + context: Int + "Use case-insensitive search" + ignoreCase: Boolean = true + "Keyword to search the text for" + keyword: String! + "Use a regex keyword to find matches in the text" + regex: Boolean + "ID of the submission's input files to search" + subfileId: Int + "Index of the submission's input files to search" + subfileIndex: Int + ): [TextSearchResult] + "Current status of the submission process" + status: SubmissionStatus + "Datetime the submission was updated" + updatedAt: DateTime + "ID of the user who updated the submission" + updatedBy: Int + "ID of the workflow associated with the submission" + workflowId: Int } -type AnnotationClassMetrics { - name: String - metrics: [PerClassSeqMetrics] +type SubmissionComponentStatusObject { + color: ComponentStatusColor + status: SubmissionComponentStatus + statusMessage: String } -type PerClassSeqMetrics { - "Type of span the metric is calculated on, e.g. Token, Sequence Superset" - spanType: String - "Of the predicted true positives, the percentage that were actually correct" - precision: Float - "Of the total true positives, the percentage were recovered by the model as true positives" - recall: Float - "Harmonic mean of precision and recall" - f1Score: Float - "# of examples that were predicted affirmative in the class but negative" - falsePositives: Int - "# of examples that were affirmative but were not predicted as such by the model" - falseNegatives: Int - "# of examples that were predicted affirmative and were actually affirmative" - truePositives: Int +type SubmissionCounts { + complete: Int + failed: Int + pendingAdminReview: Int + pendingAutoReview: Int + pendingReview: Int + processing: Int } -type ModelLevelMetrics { - "Type of span the metric is calculated on, e.g. Token, Sequence Superset" - spanType: String - "F1 score calculated based on pooling instances across classes" - microF1: Float - "F1 score calculated per-class and then averaged" - macroF1: Float - "F1 score calculated per-class and then weighted averaged, weighted by instances per class" - weightedF1: Float +""" +Creates an unique ID as a string, so that GraphQL will display +all of the events in a changelog +""" +type SubmissionEvent { + "Internal field for review load" + AutoReviewLoaded: Boolean + "Latest auto review for submission" + autoReview: Review + "Datetime the submission reached a completed state" + completedAt: DateTime + "Datetime the submission was created" + createdAt: DateTime + "ID of the user who created the submission" + createdBy: Int + "ID of the dataset associated with the submission" + datasetId: Int + "DEPRECATED: Submission files have been deleted from file store" + deleted: Boolean @deprecated(reason: "Now uses `files_deleted`") + "Errors occurred during this submission" + errors: String + "Submission files have been deleted from file store" + filesDeleted: Boolean + "Unique combination of ID and updated_at" + id: String + "Local URL to first stored input" + inputFile: String + "Original name of first file" + inputFilename: String + inputFiles: [SubmissionFile] + "OCR engine used for submission" + ocrEngine: String + outputFiles: [SubmissionOutput] + "Local URL to most recently stored output" + resultFile: String + retries: [SubmissionRetry] + "Submission has been marked as having been retrieved" + retrieved: Boolean + "Current status of the submission process" + status: SubmissionStatus + "ID of the submission" + submissionId: Int + "Datetime the submission was updated" + updatedAt: DateTime + "ID of the user who updated the submission" + updatedBy: Int + "ID of the workflow associated with the submission" + workflowId: Int } -type AnnotationExamplesPage { - annotationExamples: [AnnotationExample] +type SubmissionEventPage { pageInfo: PageInfo + submissions: [SubmissionEvent] } -type AnnotationExample { - rowId: Int - rowIndex: Int - text: String - annotationLabels: [AnnotationLabel] - predictions: [AnnotationPrediction] - datafile: DataFile +type SubmissionFacts { + daily: SubmissionFactsDaily + startDate: Date + total: SubmissionFactsTotal + workflowId: Int } -type AnnotationLabel { - start: Int - end: Int - label: String - userId: Int +type SubmissionFactsDaily { + avgHoursOnQueue: [DailyAvg] + completed: [DailyCount] + completedExceptionQueue: [DailyCount] + completedInReview: [DailyCount] + completedReviewQueue: [DailyCount] + predictions: [DailyCount] + rejectedInReview: [DailyCount] + startDate: String + stp: StpFactsDaily + submitted: [DailyCount] + submittedAndCompletedInReview: [DailyCount] + timeOnTask: TimeOnTaskDaily + workflowId: Int } -type AnnotationPrediction { - start: Int - end: Int - label: String - text: String - confidence: Float +type SubmissionFactsTotal { + startDate: String + stp: StpFacts + submitted: Int + workflowId: Int } -type DataFile { - id: Int - deleted: Boolean - name: String - rainbowUrl: String - status: FileStatus - statusMeta: JSONString - fileHash: String +"Each submission can have 1 or more (bundled) files associated with it" +type SubmissionFile { + "Size of file in bytes" fileSize: Int - fileType: FileType - pageIds: [Int] - numPages: Int - pages(pageNums: [Int]): [DataFilePage] - failureType: FileFailureType + "Name of original file" + filename: String + "Local URL to stored input" + filepath: String + "Type of file" + filetype: FileType + "Unique ID of this file" + id: Int + "Number of pages in file" + numPages: Int + "ID of the submission this file is associated with" + submissionId: Int } -"An enumeration." -enum FileStatus { - DOWNLOADING - DOWNLOADED - EXTRACTING - EXTRACTED - PROCESSED - PROCESSING - FAILED +type SubmissionMetric { + "Number of items completed in the workflow, whether review was enabled or disabled [SUBMISSION_COMPLETED]" + completed: Int + "Number of items accepted in the exceptions queue [SUBMISSION_REVIEW_STAT]" + completedExceptionQueue: Int + "Number of items that were accepted in either the review or exceptions queue [SUBMISSION_REVIEW_STAT]" + completedInReview: Int + "Number of items accepted in the review queue [SUBMISSION_REVIEW_STAT]" + completedReviewQueue: Int + "Number of items rejected from the exceptions queue [SUBMISSION_REVIEW_STAT]" + rejectedInReview: Int + "Number of items submitted to the workflow [SUBMISSION_SUBMITTED]" + submitted: Int } -"An enumeration." -enum FileType { - CSV - PDF - EXCEL - DOC - DOCX - PPT - PPTX - PNG - JPG - TIFF - UNKNOWN +type SubmissionMetrics { + aggregate: SubmissionMetric + daily: [DailySubmissionMetric] } -type DataFilePage { +"Each submission can have 1 or more output files created during the workflow" +type SubmissionOutput { + "ID of the workflow component that made this file" + componentId: Int + "Datetime the output file was created" + createdAt: DateTime + "Local URL to stored input" + filepath: String + "Unique ID of this output" id: Int - datafileId: Int - image: String - pageNum: Int - pageInfo: String - thumbnail: String - docStartOffset: Int - docEndOffset: Int + "ID of the submission this file is associated with" + submissionId: Int } -"An enumeration." -enum FileFailureType { - SERVER - DOWNLOAD - EXTRACTION - INCOMPATIBLE_TYPE - UNSUPPORTED_TYPE - CORRUPT_IMAGE - CSV_PARSING - INCOMPATIBLE_CSV_COLUMNS - CSV_NO_URL_DS_TYPE_DOCUMENT - CSV_NO_URL_DS_TYPE_IMAGE - CSV_REQUIRES_CONTENT - CSV_MULTIPLE_URLS +type SubmissionPage { + pageInfo: PageInfo + submissions: [Submission] } -"An enumeration." -enum ANNOTATIONEXAMPLE_COLUMN_ENUM { - ROW_ID - ROW_INDEX - TEXT +""" +Server response object when submitting to a workflow containing ids +to track the submissions and the submissions themselves if requested +""" +type SubmissionResult { + "Returned if submissions are duplicates" + isDuplicateRequest: Boolean + "Returned if submissions are not recorded" + jobIds: [String] + "Returned if submissions are recorded" + submissionIds: [Int] + "List of submission objects" + submissions: [Submission] } -type ANNOTATIONTestResult { - falsePositive: [MultiTestResult] - falseNegative: [MultiTestResult] - truePositive: [MultiTestResult] - trueNegative: [MultiTestResult] - resultCounts: ResultCounts - threshold: Float - modelId: Int +type SubmissionResults { + jobId: String } -type ObjectDetectionEvaluation { - metrics: JSONString - testResults(actual: [String] threshold: Float): OBJECT_DETECTIONTestResult +type SubmissionRetry { + "Unique ID of the submission retry" + id: Int + "Errors from previous submission" + previousErrors: String + "Status of submission before it was retried" + previousStatus: SubmissionStatus + "Errors that occurred during the retrying of this submission" + retryErrors: String + "Unique ID of the associated submission" + submissionId: Int } -type OBJECT_DETECTIONTestResult { - falsePositive: [MultiTestResult] - falseNegative: [MultiTestResult] - truePositive: [MultiTestResult] - trueNegative: [MultiTestResult] - resultCounts: ResultCounts - threshold: Float - modelId: Int +type SubmitAutoReview { + jobId: String } -type RationalizedClassificationEvaluation { - metrics: ClassificationModelMetrics - confusionResult(predicted: String! actual: String!): [SingleTestResult] - testResults(actual: String threshold: Float): RATIONALIZED_CLASSIFICATIONTestResult +"Deprecated - assumes all row_indices reflect example id" +type SubmitLabels { + success: Boolean } -type RATIONALIZED_CLASSIFICATIONTestResult { - falsePositive: [SingleTestResult] - falseNegative: [SingleTestResult] - truePositive: [SingleTestResult] - trueNegative: [SingleTestResult] - resultCounts: ResultCounts - threshold: Float - modelId: Int +type SubmitLabelsV2 { + success: Boolean } -union Prediction = AnnotationPrediction | ClassificationPrediction | ClassificationMultiplePrediction | RationalizedClassificationPrediction +type TargetName { + "Whether this target name has been deleted" + active: Boolean + id: Int + labelsetId: Int + name: String + position: Int + targetType: TargetType +} -type ClassificationPrediction { - confidences: [ClassConfidence] - tokenPredictions: [TokenPrediction] - explanations: [Explanation] - label: String +"Target name changes from labelset difference object" +type TargetNameDiff { + "List of added target names" + added: [TargetName] + "Count of added target names" + addedCount: Int + "List of removed target names" + removed: [TargetName] + "Count of removed target names" + removedCount: Int } -type TokenPrediction { - confidences: [ClassConfidence] - token: _Token +"A single search result with its context" +type TextSearchResult { + context: TextSearchResultSnippet + result: TextSearchResultSnippet } -type _Token { +"A piece of text in the document with location information" +type TextSearchResultSnippet { + "Exclusive end index in text" end: Int + "Starting index in text" start: Int text: String } -type ClassificationMultiplePrediction { - labels: [String] - confidences: [ClassConfidence] - tokenPredictions: [TokenPrediction] - explanations: [Explanation] -} - -type RationalizedClassificationPrediction { - confidences: [ClassConfidence] - tokenPredictions: [TokenPrediction] - explanations: [Explanation] - label: String -} - -type TrainingProgress { - percentComplete: Float -} - -"An enumeration." -enum ModelType { - ENSEMBLE - TFIDF - STANDARD - FINETUNE - OBJECT_DETECTION - RATIONALIZED - FORM_EXTRACTION - DOCUMENT +type TimeOnTaskDaily { + exceptions: [DailyTimeOnTask] + review: [DailyTimeOnTask] } -type ClassCount { - target: String - count: Int +type TimeOnTaskMetric { + "The average amount of minutes reviewers spend on documents for this workflow, aggregated across review and exceptions queue" + avgMinsPerDoc: Float + "The average amount of minutes reviewers spend on documents for this workflow in the exceptions queue" + avgMinsPerDocExceptions: Float + "The average amount of minutes reviewers spend on documents for this workflow in the review queue" + avgMinsPerDocReview: Float } -"An enumeration." -enum LabelResolutionStrategy { - MAJORITY_VOTE_WITH_TIES - MAJORITY_VOTE_WITHOUT_TIES - UNANIMOUS - ALL +type TimeOnTaskMetrics { + aggregate: TimeOnTaskMetric + daily: [DailyTimeOnTaskMetric] } -type Example { - id: String - Type: String - LabelsetId: Int - rowIndex: Int - datarowId: Int - source: String - targets: [Target] - datafileId: Int - datapointId: Int - predictions: [Prediction] +type TokenPrediction { + confidences: [ClassConfidence] + token: _Token } -type Target { - start: Int +type TokenSpan { + ctxId: Int end: Int - label: String - userId: Int -} - -type Processor { - processorType: ProcessorType! - args: JSONString + idx: Int + pageNum: Int + start: Int + "HIGH-COST: loads the text for each example" + value: String } -"An enumeration." -enum ProcessorType { - SPLIT - CONTENT_LENGTH - LINK_CLASSIFICATION_MODEL - OUTPUT_CSV_FORMATTER - OUTPUT_JSON_FORMATTER - INPUT_OCR_EXTRACTION - INPUT_IMAGE - VALIDATION +type TrainingProgress { + percentComplete: Float } -"Cyclone Label Breakdown Response\n\nresponse = {\n \"num_empty_examples\": len(empty_sources),\n \"num_examples\": num_examples,\n \"labelset_id\": labelset.id,\n \"target_contradictions\": percent_contradictions,\n \"source_duplicates\": percent_source_duplicate,\n \"class_balance\": target_counter,\n \"warnings\": warnings,\n}" -type DataInfo { - datasetId: Int - subsetIds: [Int] - numEmptyExamples: Int - numExamples: Int - targetContradictions: Float - sourceDuplicates: Float - classBalance: ClassBalance - sourceColumnId: Int - labelsetId: Int - warnings: [String] +type UnbundlingClassMetrics { + metrics: [UnbundlingPerClassSeqMetrics] + name: String } -type ClassBalance { - all: [ClassCount] - majorityVoteWithTies: [ClassCount] - majorityVoteWithoutTies: [ClassCount] - unanimous: [ClassCount] +type UnbundlingEvaluation { + "Query for examples in test set" + examples( + "Find results after this cursor" + after: Int + "Find results before this cursor" + before: Int + "Return results in descending order" + desc: Boolean + filters: ExampleFilter + "Max number of results to return" + limit: Int = 10 + "attribute to order results by" + orderBy: ExampleOrder + "number of pages to skip" + skip: Int + ): ExamplePage + metrics: UnbundlingModelMetrics + testResults( + actual: [String] + threshold: Float + ): CLASSIFICATION_UNBUNDLINGTestResult +} + +type UnbundlingModelLevelMetrics { + "Harmonic mean of precision and recall" + f1Score: Float + "Type of model the metric is calculated on, e.g. Classify, Split" + modelType: String + "Of the predicted true positives, the percentage that were actually correct" + precision: Float + "Of the total true positives, the percentage were recovered by the model as true positives" + recall: Float + "# of actual true occurences" + support: Int } -type DatasetUser { - id: Int - userId: Int - name: String - email: String - datasetId: Int - permissions: [String] - role: Roles +type UnbundlingModelMetrics { + "Metrics for evaluating model performance per class" + classMetrics: [UnbundlingClassMetrics] + "Metrics for evaluating model performance at the model level, across classes" + modelLevelMetrics: [UnbundlingModelLevelMetrics] } -type DocBot { - id: Int - name: String - createdAt: String - createdBy: Int - updatedAt: String - updatedBy: Int - subsetId: Int - parents: [Int] - docbotType: DocBotType - processors: [Processor] - components: [ComponentInterface] +type UnbundlingPerClassSeqMetrics { + "Harmonic mean of precision and recall" + f1Score: Float + "# of examples that were affirmative but were not predicted as such by the model" + falseNegatives: Int + "# of examples that were predicted affirmative in the class but negative" + falsePositives: Int + "Of the predicted true positives, the percentage that were actually correct" + precision: Float + "Of the total true positives, the percentage were recovered by the model as true positives" + recall: Float + "Type of span the metric is calculated on, e.g. Page" + spanType: String + "# of actual true occurences" + support: Int + "# of examples that were predicted affirmative and were actually affirmative" + truePositives: Int } -"An enumeration." -enum DocBotType { - QUESTIONNAIRE - FIRE_GROUP - MODEL_GROUP - DATASET - ETL - OUTPUT +type UpdateKeywords { + keywords: [String] } -interface ComponentInterface { +"Basic user object" +type User { + acceptedTerms: Boolean + accountLockedAt: DateTime + active: Boolean + apiRefreshToken: RefreshTokenMeta + "Epoch time of confirmation of user registration" + confirmedAt: String + confirmedDate: DateTime + email: String + "What day (if set) the user will be expired" + expiresAt: DateTime id: Int - docbotId: Int - componentAppId: Int - componentType: ComponentType - reviewable: Boolean + lastUpdate: String + lastUpdateDate: DateTime + logins( + "Find results after this cursor" + after: Int + "Find results before this cursor" + before: Int + "Return results in descending order" + desc: Boolean + "Max number of results to return" + limit: Int = 20 + "attribute to order results by" + orderBy: LOGIN_COLUMN_ENUM + "number of pages to skip" + skip: Int + ): LoginPage + name: String + numManagedDatasets: Int + preferences(app: String!, keys: [String]): [Preference] + "Epoch time of user registration" + registeredAt: String + registeredDate: DateTime + scopes: [ScopeAccess] + "The stage account setup is in" + setupProgress: UserProgressStatus + uuid: String } -"An enumeration." -enum ComponentType { - QUESTIONNAIRE - FIRE_GROUP - MODEL_GROUP - DOCUMENT - RESULT - CUSTOM_RESULT +type UserChangelog { + changesMade: [UserChangeType] + datasets: [DatasetRole] + date: DateTime + enabled: Boolean + "Unique combination of date and user_id" + id: String + previousDatasets: [DatasetRole] + previousRoles: [AppRole] + previouslyEnabled: Boolean + roles: [AppRole] + updatedAt: DateTime + updatedBy: Int + updaterEmail: String + userEmail: String + userId: Int } -"An enumeration." -enum WorkflowStatus { - COMPLETE - ADDING_DATA +type UserChangelogPage { + pageInfo: PageInfo + results: [UserChangelog] } -type SubmissionCounts { - processing: Int - pendingReview: Int - pendingAdminReview: Int - pendingAutoReview: Int - complete: Int - failed: Int +type UserPage { + pageInfo: PageInfo + users: [User] } -type SubmissionFacts { - startDate: Date - workflowId: Int - daily: SubmissionFactsDaily - total: SubmissionFactsTotal +type UserSnapshot { + createdAt: DateTime + datasets: [DatasetRole] + email: String + enabled: Boolean + id: Int + name: String + roles: [AppRole] } -type SubmissionFactsDaily { - submitted: [DailyCount] - submittedAndCompletedInReview: [DailyCount] - completed: [DailyCount] - completedInReview: [DailyCount] - completedReviewQueue: [DailyCount] - completedExceptionQueue: [DailyCount] - rejectedInReview: [DailyCount] - avgHoursOnQueue: [DailyAvg] - stp: StpFactsDaily - predictions: [DailyCount] - timeOnTask: TimeOnTaskDaily +type UserSnapshotPage { + pageInfo: PageInfo + results: [UserSnapshot] } -type DailyCount { - date: Date - count: Int +type UserSummary { + appRoles: [AppRoleCount] + users: EnabledCount } -type DailyAvg { - date: Date - avg: Float +""" +Deprecated +Represents a valid action on a component or a link +""" +type ValidAction { + "The component family of the action, model/filter/transformer" + componentFamily: ComponentFamily + "The component type of the action" + componentType: ComponentType + "The specific operation, add/edit/delete" + operation: WorkflowActionOp + "The specifics of a component type" + subType: String +} + +""" +Validates credentials and configuration for Exchange Integration. Does not create an integration. +Credentials are validated before the config. Correct credentials must be passed in before config is validated. +""" +type ValidateExchangeConfigCredentials { + "if config or credentials are invalid, a short description of the reason" + error: String + "if config or credentials are invalid, details of the error" + errorDetails: String + "true if the configuration and credentials are valid" + isValid: Boolean } -type StpFactsDaily { - workflow: [DailyStp] - model: [ModelStpDailyFacts] +"Configuration for a specific validation rule" +type ValidationConfig { + "Determines how failure to validate is handled." + onFailure: ValidationActionType + "Name of the validation rule, e.g. 'TYPE_CONVERSION' or 'DATE_RANGE'" + settingName: String + "Setting value for the validation rule, e.g. {'start_date': '2020-01-01', 'end_date': '2021-01-01'}" + settingValue: GenericScalar } -type DailyStp { - reviewNumerator: Int - autoReviewNumerator: Int - reviewDenom: Int - autoReviewDenom: Int - reviewStpPct: Float - autoReviewStpPct: Float - date: Date +"The result of running a specific validation rule on some extracted text." +type ValidationResult { + "Description of the validation error that occurred (if status is not SUCCESS)." + errorMessage: String + "Indicates whether or not the validation rule succeeded, and on failure returns the severity level indicated on validation rule setup" + validationStatus: ValidationStatus + "Name of the validation rule, e.g. 'TYPE_CONVERSION' or 'DATE_RANGE'" + validationType: String } -type ModelStpDailyFacts { +type Workflow { + "DEPRECATED: Status of the Auto review queue" + autoReviewEnabled: Boolean + @deprecated(reason: "Now uses settings' `auto_review_queue_enabled`") + componentBlueprints(componentType: ComponentType): [ComponentBlueprint] + componentLinks: [ComponentLink] + components: [ComponentInterface] + "Time the workflow was created at" + createdAt: String + "ID of the user who created the workflow" + createdBy: Int + dataset: Dataset + "ID of the dataset the workflow is associated with" + datasetId: Int + "Estimated human time to complete the workflow in minutes" + estHumanTimeMins: Int + "Field types available for the workflow" + fieldTypes( + name: String + "If provided, return only fields compatible with this target type." + targetType: TargetType + "If provided, return only fields compatible with this task type." + taskType: TaskType + ): [WorkflowFieldType] + "Fields associated with the workflow and their relationships to workflow components" + fields: [WorkflowField] + "Unique ID of the workflow" + id: Int + metrics(endDate: Date, startDate: Date): WorkflowMetrics + metricsStartDate: String + "Workflow name" name: String - modelGroupId: Int - stps: [DailyStp] - classStps: [ClassStpFacts] - "Review STP for model aggregated by parent filter context." - reviewStpForModel: Float - "Auto Review STP for model aggregated by parent filter context." - autoReviewStpForModel: Float + "Target names not used in any filters and that can be removed from the workflow." + removableTargetNames: [TargetName] + "DEPRECATED: Status of the Review queue" + reviewEnabled: Boolean + @deprecated(reason: "Now uses settings' `review_queue_enabled`") + "Whether or not the workflow can be reviewed" + reviewable: Boolean + reviewableModelGroups: [ModelGroup] + settings: ReviewSettings + "Current status of the workflow" + status: WorkflowStatus + submissionCounts: SubmissionCounts + submissionFacts: SubmissionFacts + "Whether the workflow is able to process submissions" + submissionRunnable: Boolean + "Reason why the workflow is unreviewable" + unreviewableReason: String + "Reason why the workflow is no runnable" + unrunnableReason: String + "Time the workflow was last updated" + updatedAt: String + "ID of the user who last updated the workflow" + updatedBy: Int + userRole: Roles } -type ClassStpFacts { - className: String - stps: [DailyStp] - "Review STP for class aggregated by parent filter context." - reviewStpForClass: Float - "Auto Review STP for class aggregated by parent filter context" - autoReviewStpForClass: Float +""" +A workflow blueprint represents a template from which +an entire workflow and its components can be created. +""" +type WorkflowBlueprint { + "Description of the workflow blueprint" + description: String + "Whether the workflow blueprint is enabled" + enabled: Boolean + "ID of the workflow blueprint" + id: Int + "Name of the workflow blueprint" + name: String + "Type of the workflow blueprint, system or user" + type: WorkflowBlueprintType + "UUID of the workflow blueprint" + uuid: String } -type TimeOnTaskDaily { - review: [DailyTimeOnTask] - exceptions: [DailyTimeOnTask] +type WorkflowBlueprintPage { + pageInfo: PageInfo + workflowBlueprints: [WorkflowBlueprint] } -type DailyTimeOnTask { - date: Date - minutes: Float - numReviews: Int +type WorkflowField { + "The datatype of the field (e.g. 'date', 'price', etc.)" + datatype: String + "Configuration dictionary that controls how the normalized type is displayed in Indico Review UI" + formatConfig: GenericScalar + "Field ID" + id: Int + "Configuration dictionary that controls what inputs are recognized as valid instances of a given type" + inputConfig: GenericScalar + "Associated workflow model group components and target names" + links: [WorkflowFieldLink] + "Denotes whether a single field instance or multiple field instances are expected for an associated field. For instance, an invoice may have multiple 'Line Item Total's (multiple=True), but would be expected to only have a single 'Invoice Date' field (multiple=False)" + multiple: Boolean + "Field name displayed in review UI and the workflow result file" + name: String + "Controls whether or not a valid instance of a field must be present to submit an review." + required: Boolean + "Configuration that controls which additional validation checks should be run and what actions should be taken in case of their failure." + validationConfig: [ValidationConfig] + "Workflow ID" + workflowId: Int } -type SubmissionFactsTotal { - submitted: Int - stp: StpFacts +"Link representing the association between a workflow component and a workflow field" +type WorkflowFieldLink { + "Component ID" + componentId: Int + "Field ID" + fieldId: Int + "Workflow field link ID" + id: Int + "ID of target name associated with a model component. Not provided for custom models or external models." + targetId: Int + "Target class name associated with a model component" + targetName: String + "ID of target name associated with a model component. Not provided for custom models or external models." + targetNameId: Int + "Workflow ID" + workflowId: Int } -type StpFacts { - model: [ModelStp] +"A datatype that can be used to normalize a given field (e.g. 'date', 'price', 'general', etc.)" +type WorkflowFieldType { + "The schema and defaults for field `format_config`. Defines settings that control how normalized fields are formatted in the Review UI." + formatConfigSchema: GenericScalar + "The schema and defaults for field `input_config`. Defines settings that control which text extractions are considered valid instances of a field type." + inputConfigSchema: GenericScalar + "The name of the field datatype (e.g. 'date', 'price', etc.)" + name: String + "Target type corresponding to this field type." + targetType: TargetType + validationConfigSchema: GenericScalar + "The ID of the workflow the field type is associated with (if using a custom field type)." + workflowId: Int } -type ModelStp { - reviewNumerator: Int - autoReviewNumerator: Int - reviewDenom: Int - autoReviewDenom: Int - reviewStpPct: Float - autoReviewStpPct: Float - name: String - modelGroupId: Int +type WorkflowGallery { + blueprintsPage( + "Find results after this cursor" + after: Int + "Find results before this cursor" + before: Int + "Return results in descending order" + desc: Boolean + filters: WorkflowBlueprintFilter + "Max number of results to return" + limit: Int + "attribute to order results by" + orderBy: WORKFLOWBLUEPRINT_COLUMN_ENUM + "number of pages to skip" + skip: Int + ): WorkflowBlueprintPage } type WorkflowMetrics { - workflowId: Int - startDate: Date endDate: Date - timeOnTask: TimeOnTaskMetrics - submissions: SubmissionMetrics - queues: QueueMetrics - predictions: PredictionMetrics - straightThroughProcessing: StpMetrics "The first date an item was submitted to this workflow" firstSubmittedDate: Date + predictions: PredictionMetrics + queues: QueueMetrics + startDate: Date + straightThroughProcessing: StpMetrics + submissions: SubmissionMetrics + tagsAvailable: [WorkflowMetricsTag] + timeOnTask: TimeOnTaskMetrics + workflowId: Int } -type TimeOnTaskMetrics { - aggregate: TimeOnTaskMetric - daily: [DailyTimeOnTaskMetric] +type WorkflowPage { + pageInfo: PageInfo + workflows: [Workflow] } -type TimeOnTaskMetric { - "The average amount of minutes reviewers spend on documents for this workflow, aggregated across review and exceptions queue" - avgMinsPerDoc: Float - "The average amount of minutes reviewers spend on documents for this workflow in the review queue" - avgMinsPerDocReview: Float - "The average amount of minutes reviewers spend on documents for this workflow in the exceptions queue" - avgMinsPerDocExceptions: Float +type WorkflowStpMetrics { + "Daily STP metrics aggregated to the level of the workflow" + daily: [DailyStpMetric] } -type DailyTimeOnTaskMetric { - date: Date - "The average amount of minutes reviewers spend on documents for this workflow, aggregated across review and exceptions queue" - avgMinsPerDoc: Float - "The average amount of minutes reviewers spend on documents for this workflow in the review queue" - avgMinsPerDocReview: Float - "The average amount of minutes reviewers spend on documents for this workflow in the exceptions queue" - avgMinsPerDocExceptions: Float +type _ClassLabel { + clsName: TargetName } -type SubmissionMetrics { - aggregate: SubmissionMetric - daily: [DailySubmissionMetric] +type _ClassPred { + classConfidences: [ClassConfidenceV2] + clsName: TargetName } -type SubmissionMetric { - "Number of items submitted to the workflow" - submitted: Int - "Number of items completed in the workflow, whether review was enabled or disabled" - completed: Int - "Number of items that were accepted in either the review or exceptions queue" - completedInReview: Int - "Number of items rejected from the exceptions queue" - rejectedInReview: Int - "Number of items accepted in the review queue" - completedReviewQueue: Int - "Number of items accepted in the exceptions queue" - completedExceptionQueue: Int +type _ExtractLabel { + clsName: TargetName + end: Int + pageNum: Int + start: Int } -type DailySubmissionMetric { - date: Date - "Number of items submitted to the workflow" - submitted: Int - "Number of items completed in the workflow, whether review was enabled or disabled" - completed: Int - "Number of items that were accepted in either the review or exceptions queue" - completedInReview: Int - "Number of items rejected from the exceptions queue" - rejectedInReview: Int - "Number of items accepted in the review queue" - completedReviewQueue: Int - "Number of items accepted in the exceptions queue" - completedExceptionQueue: Int +type _ExtractionPred { + classConfidences: [ClassConfidenceV2] + clsName: TargetName + end: Int + pageNum: Int + start: Int } -type QueueMetrics { - dailyCumulative: [DailyQueueMetric] +type _MultiClassLabel { + clsNames: [TargetName] +} + +type _MultiClassPred { + classConfidences: [ClassConfidenceV2] + clsNames: [TargetName] } -type DailyQueueMetric { - date: Date - "Number of submissions on the queues waiting to be reviewed" - subsOnQueue: Int - "Cumulative hours of items on queues waiting to be reviewed" - hoursOnQueue: Float - "Average cumulative age of items on queues waiting to be reviewed" - avgAgeInQueue: Float +type _Token { + end: Int + start: Int + text: String } -type PredictionMetrics { - "Total number of model generated predictions for this workflow" - aggregate: PredictionMetric - "Number of model generated predictions for the workflow, each day" - daily: [DailyPredictionMetric] +"An enumeration." +enum AddDataComponentStatus { + FAILED + NOT_READY + READY } -type PredictionMetric { - numPreds: Int +"Roles for App Users and the mapping of permissions" +enum AppRole { + APP_ADMIN + CELERY_FLOWER + MANAGE_ALL_DATA + REPORT_GENERATOR + TEAM_ADMIN + TEAM_DEVELOPER + TEAM_USER } -type DailyPredictionMetric { - date: Date - numPreds: Int +"An enumeration." +enum BLUEPRINTASSOCIATEDWORKFLOW_COLUMN_ENUM { + DATASET_ID + ID + NAME } -type StpMetrics { - "STP metrics aggregate at the level of the workflow" - workflow: WorkflowStpMetrics - "Schema for model STP metrics including class STP metrics as child nodes on this object's schema" - model: [ModelStpMetrics] +"An enumeration." +enum BlueprintOP { + CREATE + DELETE + LIST_WORKFLOWS +} + +"Tags for all blueprints" +enum BlueprintTag { + accelerator + base64ai + classification + custom + extraction + forms + huggingface + imported + indico + microsoft + openai + privateai + static + trainable } -type WorkflowStpMetrics { - "Daily STP metrics aggregated to the level of the workflow" - daily: [DailyStpMetric] +"An enumeration." +enum COMMATTEMPT_COLUMN_ENUM { + ERROR + FINISHED + ID + STARTED + SUCCESS } -type DailyStpMetric { - date: Date - "The number of human accepted model predictions that exactly match model predictions" - reviewNumerator: Int - "The number of human accepted auto review labels" - autoReviewNumerator: Int - "The union of user supplied labels and model predictions" - reviewDenom: Int - "The union of user supplied labels and auto review labels" - autoReviewDenom: Int - "Review numerator divided by review denominator, applicable if review is enabled and auto-review is disabled" - reviewStpPct: Float - "Auto review numerator divided by auto review denomoinator, applicable if auto-review is enabled" - autoReviewStpPct: Float +"An enumeration." +enum COMPONENTBLUEPRINT_COLUMN_ENUM { + COMPONENT_FAMILY + COMPONENT_TYPE + DESCRIPTION + ENABLED + FOOTER + ICON + ID + NAME + PROVIDER } -type ModelStpMetrics { - modelGroupId: Int - name: String - "Aggregate STP metrics for the model based on filters applied to the query" - aggregate: StpMetric - "Daily STP metrics for the model" - daily: [DailyStpMetric] - "STP metrics for each class associated with the model" - classMetrics: [ClassStpMetrics] +"Groups for component types" +enum ComponentFamily { + FILTER + MODEL + OUTPUT + REVIEW + TRANSFORMER } -type StpMetric { - "The number of human accepted model predictions that exactly match model predictions" - reviewNumerator: Int - "The number of human accepted auto review labels" - autoReviewNumerator: Int - "The union of user supplied labels and model predictions" - reviewDenom: Int - "The union of user supplied labels and auto review labels" - autoReviewDenom: Int - "Review numerator divided by review denominator, applicable if review is enabled and auto-review is disabled" - reviewStpPct: Float - "Auto review numerator divided by auto review denomoinator, applicable if auto-review is enabled" - autoReviewStpPct: Float +"An enumeration." +enum ComponentStatusColor { + GREEN + RED + YELLOW } -type ClassStpMetrics { - className: String - "STP metrics for this class, aggregated based on the filters applied to the query" - aggregate: StpMetric - "STP metrics for this class, daily" - daily: [DailyStpMetric] +"An enumeration." +enum ComponentType { + BLUEPRINT + CONTENT_LENGTH + CUSTOM_FILTER + CUSTOM_MODEL + CUSTOM_OUTPUT + CUSTOM_RESULT + CUSTOM_TRANSFORMER + DEFAULT_OUTPUT + DOCUMENT + EXTERNAL_MODEL + INPUT_IMAGE + INPUT_OCR_EXTRACTION + LINK_CLASSIFICATION_MODEL + LINK_LABEL + MODEL_GROUP + OUTPUT_CSV_FORMATTER + OUTPUT_JSON_FORMATTER + QUESTIONNAIRE + RESULT + REVIEW + SPLIT + VALIDATION } "An enumeration." -enum WORKFLOW_COLUMN_ENUM { - ID - DATASET_ID +enum DATASET_COLUMN_ENUM { CREATED_AT CREATED_BY - UPDATED_AT - UPDATED_BY + DEFAULT_DATACOLUMN_ID + DEFAULT_SUBSET_ID + ERROR_INFO + ID NAME - USER_ROLE + NUM_MODEL_GROUPS + NUM_QUESTIONNAIRES + ROW_COUNT STATUS - REVIEWABLE - REVIEW_ENABLED - AUTO_REVIEW_ENABLED - EST_HUMAN_TIME_MINS + TYPE + UPDATED_AT + UPDATED_BY } -input WorkflowFilter { - "name contains" - name: String - "the workflow can use Review for its submissions" - reviewable: Boolean - "all new submissions will pass through Review" - reviewEnabled: Boolean - "all new submissions will wait for Auto Review" - autoReviewEnabled: Boolean - OR: [WorkflowFilter] - AND: [WorkflowFilter] - ors: [WorkflowFilter] - ands: [WorkflowFilter] +enum DataType { + IMAGE + TEXT } -type Submission { - id: Int - datasetId: Int - workflowId: Int - status: SubmissionStatus - "Errors occured during this submission" - errors: String - "Submission files have been deleted from file store" - deleted: Boolean - inputFiles: [SubmissionFile] - "Local URL to first stored input" - inputFile: String - "Original name of first file" - inputFilename: String - "Local URL to stored output" - resultFile: String - "Submission has been marked as having been retrieved" - retrieved: Boolean - "Latest auto review for submission" - autoReview: Review - "Internal field for review load" - AutoReviewLoaded: Boolean +"An enumeration." +enum DataTypes { + CATEGORICAL + IMAGE + NUMERIC + STRING + UNKNOWN } -"An enumeration." -enum SubmissionStatus { - PROCESSING - PENDING_AUTO_REVIEW - PENDING_REVIEW - PENDING_ADMIN_REVIEW +"Status enums for Datasets" +enum DatasetStatus { COMPLETE + CREATING + DELETING FAILED + PROCESSED + STAGED + UPLOADING } -type SubmissionFile { - id: Int - "Local URL to stored input" - filepath: String - "Name of original file" - filename: String - submissionId: Int +"An enumeration." +enum DatasetType { + DOCUMENT + IMAGE + TEXT } -type Review { - id: Int - submissionId: Int - createdAt: String - createdBy: Int - completedAt: String - rejected: Boolean - adminReview: Boolean @deprecated(reason: "Please use review_type") - reviewType: ReviewType - notes: String - changes: JSONString +"An enumeration." +enum EXPORT_COLUMN_ENUM { + ANONYMOUS + CREATED_AT + CREATED_BY + DATASET_ID + DOWNLOAD_URL + ID + LABELSET_ID + NAME + NUM_LABELS + STATUS +} + +enum ExampleOrder { + DATAFILE_NAME + ID + STATUS + UPDATED_AT } "An enumeration." -enum ReviewType { - MANUAL - AUTO - ADMIN +enum ExampleStatus { + COMPLETE + INCOMPLETE + REJECTED } -type SubmissionPage { - submissions: [Submission] - pageInfo: PageInfo +"Status enums for Exports" +enum ExportStatus { + COMPLETE + FAILED + STARTED } "An enumeration." -enum SUBMISSION_COLUMN_ENUM { - ID - DATASET_ID - WORKFLOW_ID - STATUS - ERRORS - DELETED - INPUT_FILE - INPUT_FILENAME - RESULT_FILE - RETRIEVED - _AUTO_REVIEW_LOADED +enum FeatureDomainEnum { + EMOTION + ENSEMBLE + FASTTEXT + FINANCE + IMAGE_ENSEMBLE + IMAGE_V2 + IMAGE_V3 + IMAGE_V4 + SENTIMENT + STANDARD + STANDARD_V2 + TOPICS + UNSUPERVISEDSENTIMENT +} + +"An enumeration." +enum FileFailureType { + CORRUPT_IMAGE + CSV_MULTIPLE_URLS + CSV_NO_URL_DS_TYPE_DOCUMENT + CSV_NO_URL_DS_TYPE_IMAGE + CSV_PARSING + CSV_REQUIRES_CONTENT + CSV_TOO_MANY_ROWS + CSV_TOO_MANY_TARGETS + DOWNLOAD + EMPTY_FILE + EXTRACTION + INCOMPATIBLE_CSV_COLUMNS + INCOMPATIBLE_TYPE + PASSWORD_PROTECTED + SERVER + TOO_LARGE + TOO_MANY_PAGES + UNSUPPORTED_TYPE } -input SubmissionFilter { - "input filename contains" - inputFilename: String - "submission status is" - status: SubmissionStatus - "Submission has been marked as having been retrieved" - retrieved: Boolean - OR: [SubmissionFilter] - AND: [SubmissionFilter] - ors: [SubmissionFilter] - ands: [SubmissionFilter] +"An enumeration." +enum FileStatus { + DOWNLOADED + DOWNLOADING + EXTRACTED + EXTRACTING + FAILED + PROCESSED + PROCESSING } -type Refresh { - refreshedAt: DateTimeISO - refreshStartedAt: DateTimeISO - refreshing: Boolean +"Enum for file types" +enum FileType { + CSV + DOC + DOCX + EML + EXCEL + JPG + MSG + PDF + PNG + PPT + PPTX + RTF + TIFF + TXT + UNKNOWN + XLS + XLSX } -type QuestionnairePage { - questionnaires: [Questionnaire] - pageInfo: PageInfo +"An enumeration." +enum IntegrationStatus { + ACTIVE + FAILED + PAUSED } -type Questionnaire { - id: Int - "Labeling tasks are enabled" - active: Boolean - "Predictions are enabled" - activeLearning: Boolean - role: Roles - assignedUsers: [QuestionnaireUser] - "Unix timestamp" - createdAt: String - "User id" - createdBy: Int - dataType: DataType - datasetId: Int - subsetId: Int - "Labeling will always be done in Text mode" - forceTextMode: Boolean - instructions: String - name: String - numTotalExamples: Int - numFullyLabeled: Int - numLabeledByMe: Int - numRejected: Int - processors: [Processor] - "On-document labeling interface enabled" - odl: Boolean - questions: [Question] - "cumulative status of all questions in questionnaire" - questionsStatus: QuestionStatus - sourceColumnId: Int - "Unix timestamp" - updatedAt: String - "User id" - updatedBy: Int - examples(numExamples: Int! "First question in questionnaire is selected if not provided" questionId: Int datafileId: Int): [Example] +"An enumeration." +enum IntegrationType { + EXCHANGE } -type QuestionnaireUser { - id: Int - userId: Int - email: String - name: String - questionnaireId: Int - permissions: [String] - role: Roles - createdAt: String - createdBy: Int - datasetId: Int - labelCount: Int +"Adapted from Celery Task Status" +enum JobStatus { + FAILURE + IGNORED + PENDING + RECEIVED + REJECTED + RETRY + REVOKED + STARTED + SUCCESS + TRAILED } -enum DataType { - TEXT - IMAGE +"An enumeration." +enum LOGIN_COLUMN_ENUM { + ID + LOGIN_IP } -type Question { - id: Int - type: TaskType - DatasetId: Int - SubsetId: Int - labelsetId: Int - labelset: LabelSet - keywords: [String] - status: QuestionStatus - questionnaireId: Int - text: String - modelGroupId: Int - modelGroup: ModelGroup - "Ordered list of target names." - targets: [String] +"An enumeration." +enum LabelLinkStrategy { + BY_KEY + BY_ROW } "An enumeration." -enum QuestionStatus { - STARTED - FAILED - COMPLETE +enum LabelResolutionStrategy { + ALL + MAJORITY_VOTE_WITHOUT_TIES + MAJORITY_VOTE_WITH_TIES + UNANIMOUS } "An enumeration." -enum QUESTIONNAIRE_COLUMN_ENUM { - ID - ACTIVE - ACTIVE_LEARNING - ROLE +enum MODELGROUP_COLUMN_ENUM { + COMPONENT_ID CREATED_AT CREATED_BY - DATA_TYPE DATASET_ID - SUBSET_ID - FORCE_TEXT_MODE - INSTRUCTIONS + DATA_TYPE + ID + INTERLABELER_RESOLUTION + LABELSET_COLUMN_ID NAME - NUM_TOTAL_EXAMPLES - NUM_FULLY_LABELED - NUM_LABELED_BY_ME - NUM_REJECTED - ODL - QUESTIONS_STATUS + QUESTIONNAIRE_ID + RETRAIN_REQUIRED + SELECTED_MODEL_ID SOURCE_COLUMN_ID + STATUS + SUBSET_ID + TASK_TYPE UPDATED_AT UPDATED_BY + WORKFLOW_ID } -input QuestionnaireFilter { - "name contains" - name: String - OR: [QuestionnaireFilter] - AND: [QuestionnaireFilter] - ors: [QuestionnaireFilter] - ands: [QuestionnaireFilter] +"An enumeration." +enum ModelStatus { + COMPLETE + CREATING + FAILED + NOT_ENOUGH_DATA + TRAINING } -type ModelGroupPage { - modelGroups: [ModelGroup] - pageInfo: PageInfo +"An enumeration." +enum ModelType { + DOCUMENT + ENSEMBLE + FINETUNE + FORM_EXTRACTION + OBJECT_DETECTION + RATIONALIZED + STANDARD + TABLE + TFIDF + TFIDF_GBT + TFIDF_LR + UNBUNDLE } "An enumeration." -enum MODELGROUP_COLUMN_ENUM { - ID - DATASET_ID - NAME - STATUS - TASK_TYPE - DATA_TYPE - RETRAIN_REQUIRED - INTERLABELER_RESOLUTION - CREATED_AT - CREATED_BY - UPDATED_AT - UPDATED_BY - SUBSET_ID - LABELSET_COLUMN_ID +enum OCREngine { + OMNIPAGE + READAPI + READAPI_TABLES_V1 + READAPI_V2 } -input ModelGroupFilter { - "name contains" - name: String - "model group task type is" - taskType: TaskType - "model group subset id is" - subsetId: Int - OR: [ModelGroupFilter] - AND: [ModelGroupFilter] - ors: [ModelGroupFilter] - ands: [ModelGroupFilter] +"Permissions that are mapped to AppRoles" +enum PermissionType { + ADD_ADMIN_REVIEW + ADD_LABEL + ADD_REVIEW + CREATE_SUBMISSION + DELETE_DATASET + FEATURIZE + MANAGE_USERS + MODIFY_METADATA + READ_DATAPOINTS + READ_LABELS + READ_METADATA + READ_SUBMISSIONS + READ_USERS } -type ModelSimilarity { - row: Int - value: Int +""" +DEPRECATED: but kept around for old migrations +""" +enum ProcessorType { + CONTENT_LENGTH + INPUT_IMAGE + INPUT_OCR_EXTRACTION + LINK_CLASSIFICATION_MODEL + OUTPUT_CSV_FORMATTER + OUTPUT_JSON_FORMATTER + SPLIT + VALIDATION } -type Job { - id: String - status: JobStatus - result: JSONString - ready: Boolean +"An enumeration." +enum QUESTIONNAIRE_COLUMN_ENUM { + ACTIVE + ACTIVE_LEARNING + CREATED_AT + CREATED_BY + DATASET_ID + DATA_TYPE + FORCE_TEXT_MODE + ID + INSTRUCTIONS + LABELSET_ID + MODEL_GROUP_ID + NAME + NUM_FULLY_LABELED + NUM_LABELED_BY_ME + NUM_REJECTED + NUM_TOTAL_EXAMPLES + ODL + QUESTIONS_STATUS + ROLE + SHOW_PREDICTIONS + SOURCE_COLUMN_ID + STATUS + SUBSET_ID + UPDATED_AT + UPDATED_BY } -"Adapted from Celery Task Status" -enum JobStatus { - "Task state is unknown (assumed pending since you know the id)." - PENDING - "Task was received by a worker (only used in events)." - RECEIVED - "Task was started by a worker (:setting:`task_track_started`)." +"Status enums for Questions" +enum QuestionStatus { + COMPLETE + FAILED STARTED - "Task succeeded" - SUCCESS - "Task failed" - FAILURE - "Task was revoked." - REVOKED - "Task was rejected (only used in events)." - REJECTED - "Task is waiting for retry." - RETRY - "Job Status IGNORED" - IGNORED - "Job Status TRAILED" - TRAILED } -type FireGroupPage { - fireGroups: [FireGroup] - pageInfo: PageInfo +"Supported formats for reports" +enum ReportFormat { + CSV + JSON } -type FireGroup { - id: Int - createdAt: String - createdBy: Int - datacolumnId: Int - datasetId: Int - defaultFire: Fire - featurecolumnId: Int - fires: [Fire] - name: String - numRows: Int - processors: [Processor] - ready: Boolean - status: FireStatus - subsetId: Int - updatedAt: String - updatedBy: Int +"An enumeration." +enum ReviewType { + ADMIN + AUTO + MANUAL } -type Fire { - id: Int - clusterId: Int - createdAt: String - createdBy: Int - datacolumnId: Int - datasetId: Int - featurecolumnId: Int - filters: String - fireGroupId: Int - status: FireStatus - subsetId: Int - vectorizerId: Int +"An enumeration." +enum RocAucAveraging { + SIMPLE + WEIGHTED } -"An enumeration." -enum FireStatus { - CREATING - COMPLETE - FAILED +enum Roles { + ANALYST + LABELER + LABELER_AND_REVIEWER + MANAGER + REVIEWER } "An enumeration." -enum FIREGROUP_COLUMN_ENUM { - ID +enum SUBMISSIONEVENT_COLUMN_ENUM { + COMPLETED_AT CREATED_AT CREATED_BY - DATACOLUMN_ID DATASET_ID - FEATURECOLUMN_ID - NAME - NUM_ROWS - READY + DELETED + ERRORS + FILES_DELETED + ID + INPUT_FILE + INPUT_FILENAME + OCR_ENGINE + RESULT_FILE + RETRIEVED STATUS - SUBSET_ID + SUBMISSION_ID UPDATED_AT UPDATED_BY + WORKFLOW_ID + _AUTO_REVIEW_LOADED } -input FireGroupFilter { - "name contains" - name: String - OR: [FireGroupFilter] - AND: [FireGroupFilter] - ands: [FireGroupFilter] - ors: [FireGroupFilter] +"An enumeration." +enum SUBMISSION_COLUMN_ENUM { + COMPLETED_AT + CREATED_AT + CREATED_BY + DATASET_ID + DELETED + ERRORS + FILES_DELETED + ID + INPUT_FILE + INPUT_FILENAME + OCR_ENGINE + RESULT_FILE + RETRIEVED + REVIEW_IN_PROGRESS + STATUS + UPDATED_AT + UPDATED_BY + WORKFLOW_ID + _AUTO_REVIEW_LOADED } -type FireSimilarity { - row: Int - value: Int +"An enumeration." +enum SamplingStrategy { + NO_SAMPLING + RANDOM_OVERSAMPLE } -type ExportPage { - exports: [Export] - pageInfo: PageInfo +"User Scopes" +enum Scope { + ALL_SUBMISSION_LOGS + ALL_USER_REPORTS + APP_ACCESS + BASE + CELERY_FLOWER + CHANGE_PASSWORD + CONFIRM_ACCOUNT + GRAPHIQL + MANAGE_ALL_DATA + MANAGE_DATASET + MANAGE_USERS + METRICS + REFRESH_TOKEN + USER_INFORMATION } "An enumeration." -enum EXPORT_COLUMN_ENUM { - ID - DATASET_ID - NAME - STATUS - NUM_LABELS - ANONYMOUS - DOWNLOAD_URL - CREATED_AT - CREATED_BY +enum SubmissionComponentStatus { + CANNOT_EXECUTE + CAN_EXECUTE } "An enumeration." -enum PermissionType { - READ_METADATA - READ_LABELS - READ_DATAPOINTS - MODIFY_METADATA - DELETE_DATASET - ADD_LABEL - MANAGE_USERS - FEATURIZE - READ_USERS - READ_SUBMISSIONS - ADD_REVIEW - CREATE_SUBMISSION - ADD_ADMIN_REVIEW +enum SubmissionResultVersion { + LATEST + OLDEST_SUPPORTED + ONE + THREE + TWO } -type DatasetPage { - datasets: [Dataset] - pageInfo: PageInfo +"An enumeration." +enum SubmissionStatus { + COMPLETE + FAILED + PENDING_ADMIN_REVIEW + PENDING_AUTO_REVIEW + PENDING_REVIEW + PROCESSING } "An enumeration." -enum DATASET_COLUMN_ENUM { - ID - NAME - STATUS - ROW_COUNT - DEFAULT_SUBSET_ID - TYPE - ERROR_INFO - CREATED_AT - CREATED_BY - UPDATED_AT - UPDATED_BY - NUM_FIRE_GROUPS - NUM_MODEL_GROUPS - NUM_QUESTIONNAIRES +enum TableReadOrder { + COLUMN + ROW } -input DatasetFilter { - "name contains" - name: String - OR: [DatasetFilter] - AND: [DatasetFilter] - ands: [DatasetFilter] - ors: [DatasetFilter] +"An enumeration." +enum TargetType { + ANCHOR + CATEGORY + CHECKBOX + CROP + PAGE_RANGE + RADIO + SIGNATURE + TEXT } -type DataPointSearchResult { - result: DataPointSearchResultSnippet - context: DataPointSearchResultSnippet +"An enumeration." +enum TaskType { + ANNOTATION + CLASSIFICATION + CLASSIFICATION_MULTIPLE + CLASSIFICATION_UNBUNDLING + FORM_EXTRACTION + OBJECT_DETECTION + RATIONALIZED_CLASSIFICATION + REGRESSION } -type DataPointSearchResultSnippet { - "Starting index in text" - start: Int - "Exclusive end index in text" - end: Int - text: String +"An enumeration." +enum USERCHANGELOG_COLUMN_ENUM { + DATE + ENABLED + ID + PREVIOUSLY_ENABLED + UPDATED_AT + UPDATED_BY + UPDATER_EMAIL + USER_EMAIL + USER_ID } -type SuperMutation { - generateResetLink(userId: Int!): GenerateResetLink - activateUser(id: Int!): User - deactivateUser(id: Int!): User - modifyScopes(id: Int! scopes: [Scope]!): User - unlockUser(id: Int!): User - invalidateSessions(id: Int!): User - workflowSubmission("Batch all files under a single submission" bundle: Boolean = false "UUID for duplicate Submissions caching" duplicationId: String files: [FileInput]! "Record submission for future use" recordSubmission: Boolean = true "Submission output result file version" resultVersion: SubmissionResultVersion workflowId: Int!): SubmissionResult - workflowUrlSubmission("Batch all urls under a single submission" bundle: Boolean = false "UUID for duplicate Submissions caching" duplicationId: String "Record submission for future use" recordSubmission: Boolean = true "Submission output result file version" resultVersion: SubmissionResultVersion urls: [String]! workflowId: Int!): SubmissionResult - toggleWorkflowReview("If toggling review off, mark existing submissions waiting for review as complete. Ignored if toggling review on." completeExistingSubmissions: Boolean = false "Place all future submissions into review queue" enableReview: Boolean! workflowId: Int!): Workflow - toggleWorkflowAutoReview("All new submissions will wait for Auto Review" enableAutoReview: Boolean! "If toggling auto review on, mark existing subs pending review as pending auto review. Ignore if toggling off" updateExistingSubmissions: Boolean = false workflowId: Int!): Workflow - addDataToWorkflow(workflowId: Int!): AddDataToWorkflow - updateWorkflowMeta("Estimated human time to complete the workflow in minutes" estHumanTimeMins: Int name: String workflowId: Int!): Workflow - updateSubmission("Mark the submission as having been retrieved" retrieved: Boolean submissionId: Int!): Submission - submitReview(changes: JSONString notes: String rejected: Boolean = false submissionId: Int!): Review - submitAutoReview(changes: JSONString "Bypass Review\/Exception queue (not recommended)" forceComplete: Boolean = false rejected: Boolean = false submissionId: Int!): SubmitAutoReview - submissionResults(submissionId: Int!): SubmissionResults - userSnapshotReport("User information on this date (23:59 UTC)" date: Date filters: UserReportFilter "Format of report to generate, defaults to CSV" reportFormat: ReportFormat): GenerateUserSnapshotReport - userChangelogReport("Changelog up to this date (23:59 UTC)" endDate: Date filters: UserReportFilter "Format of report to generate, defaults to CSV" reportFormat: ReportFormat "Changelog from this date (00:00 UTC)" startDate: Date): GenerateUserChangelogReport - refreshViews("Force refresh views if the views were refreshed less than cooldown period ago." force: Boolean = false): Refresh - createQuestionnaire("Enable predictions on the questionnaire" activeLearning: Boolean = true dataType: DataType! datasetId: Int! "Always use Text Labeling UI" forceTextMode: Boolean instructions: String modelTrainingOptions: JSONString modelType: ModelType name: String! numLabelersRequired: Int! "Create a new questionnaire from an existing labelset." originalLabelsetId: Int processors: [InputProcessor] questions: [QuestionInput]! sourceColumnId: Int!): Questionnaire - updateQuestionnaire("Enable labeling tasks" active: Boolean "Enable predictions on the questionnaire" activeLearning: Boolean dataType: DataType id: Int! instructions: String name: String): Questionnaire - deleteQuestionnaire(id: Int!): DeleteQuestionnaire - submitLabels(datasetId: Int! labels: [SubmissionLabel]! labelsetId: Int! "Model group to retrain after label submission" modelGroupId: Int): SubmitLabels - addTarget(questionnaireId: Int! target: String!): Question - updateQuestionKeywords("Use keywords for all users" globalPreference: Boolean = false keywords: [String]! questionnaireId: Int!): UpdateKeywords - addQuestionnaireUser(id: Int! userId: Int!): QuestionnaireUser - removeQuestionnaireUser(id: Int! userId: Int!): RemoveQuestionnaireUser - cancelModelTraining(modelId: Int!): CancelModelTraining - createModelGroup(datasetId: Int! domain: FeatureDomainEnum finetune: Boolean interlabelerResolution: LabelResolutionStrategy labelsetColumnId: Int makePredictions: Boolean = false modelTrainingOptions: JSONString modelType: ModelType name: String! processors: [InputProcessor] rowIdx: [Int] sourceColumnId: Int! subsetId: Int testSplit: Float = 0.2): ModelGroup - retrainModelGroup(forceRetrain: Boolean interlabelerResolution: LabelResolutionStrategy modelGroupId: Int! "Can only be updated on retrain for extraction models." modelType: ModelType): ModelGroup - updateModelGroupSettings(domain: FeatureDomainEnum finetune: Boolean interlabelerResolution: LabelResolutionStrategy makePredictions: Boolean modelGroupId: Int! predictOptions: JSONString rocAucAveraging: RocAucAveraging samplingStrategy: SamplingStrategy taskType: TaskType testSplit: Float wordPredictorStrength: WordPredictorStrength): ModelGroup - updateModelGroupName(modelGroupId: Int! name: String!): ModelGroup - deleteModelGroup(modelGroupId: Int!): DeleteModelGroup - optimizeModelGroup(makePredictions: Boolean modelGroupId: Int!): ModelGroup - modelPredict(data: [String] modelId: Int! predictOptions: JSONString): ModelPredict - modelLoad(modelId: Int!): ModelLoad - updateLabelsetTargetPositions(labelsetId: Int! targetNames: [String]): LabelSet - addLabelsetTarget(labelsetId: Int! targetName: String!): LabelSet - createFireGroup(datacolumnId: Int datasetId: Int filters: [FiltersInput] name: String processors: [InputProcessor]): FireGroup - updateFireGroup(filters: [FiltersInput] id: Int! name: String): FireGroup - deleteFireGroup(id: Int!): DeleteFireGroup - pdfExtraction(data: [String]! images: Boolean metadata: Boolean pageFormat: String rawText: Boolean singleColumn: Boolean tables: Boolean text: Boolean): PDFExtraction - documentExtraction(files: [FileInput] jsonConfig: JSONString): DocumentExtraction - activeFormFields(files: [FileInput]): ActiveFormFields - createExport("Anonymize user information" anonymous: Boolean = false columnIds: [Int] "One row per example, combine labels from multiple labelers into a single row" combineLabels: Boolean = false datasetId: Int! "Include datafile information" fileInfo: Boolean = true labelsetIds: [Int] name: String subsetIds: [Int]): Export - newDataset("DEPRECATED: Use kloudless uploader. If false, uses Nginx for local file uploads" kloudless: Boolean = false metadataList: JSONString!): Dataset - updateDataset(datasetId: Int! "New name of the dataset." name: String): Dataset - addDatasetFiles("Automatically process files that are uploaded and associated with the dataset" autoprocess: Boolean = false datasetId: Int! metadataList: JSONString!): Dataset - deleteDatasetFile(datasetId: Int! fileId: Int!): Dataset - deleteDataset(id: Int!): DeleteDataset - addDatasetUser(datasetId: Int! email: String! role: Roles!): DatasetUser - modifyDatasetUser(datasetId: Int! role: Roles! userId: Int!): DatasetUser - deleteDatasetUser(datasetId: Int! userId: Int!): DeleteDatasetUser - createDataset(datasetType: DatasetType "Name of the dataset to create" name: String!): Dataset - addDataFiles(datafileIds: [Int] datasetId: Int!): Dataset - addDataCsv(datafileIds: [Int] datasetId: Int!): Dataset - retryUrl(datafileId: Int! datasetId: Int! newUrl: String!): Dataset - "Example:\nmutation update_py_user {\n updateUser(sZip:\"12345\") {\n updated\n }\n}" - updateUser(id: Int name: String sCity: String sCountry: String sState: String sStreet: String sStreet2: String sZip: String): User - "Example:\nmutation generate_new_refresh_token {\n GenerateNewApiRefreshToken{\n refresh_token\n }\n}" - generateNewApiRefreshToken: GenerateNewApiRefreshToken - updateUserPreference(app: String! key: String! value: JSONString!): Preference - deleteUserPreference(app: String! key: String!): DeletePreference +"An enumeration." +enum USERSNAPSHOT_COLUMN_ENUM { + CREATED_AT + EMAIL + ENABLED + ID + NAME } -type GenerateResetLink { - link: String +"An enumeration." +enum USER_COLUMN_ENUM { + ACCEPTED_TERMS + ACCOUNT_LOCKED_AT + ACTIVE + CONFIRMED_AT + CONFIRMED_DATE + EMAIL + EXPIRES_AT + ID + LAST_UPDATE + LAST_UPDATE_DATE + NAME + NUM_MANAGED_DATASETS + REGISTERED_AT + REGISTERED_DATE + SETUP_PROGRESS + UUID } -type SubmissionResult { - "Returned if submissions are not recorded" - jobIds: [String] - "Returned if submissions are recorded" - submissionIds: [Int] - submissions: [Submission] - "Returned if submissions are duplicates" - isDuplicateRequest: Boolean +"change types for user is modified" +enum UserChangeType { + APP_ROLE + DATASET_ROLE + ENABLEMENT } -input FileInput { - filename: String - filemeta: JSONString +enum UserProgressStatus { + COMPLETE + COPYING_DATASETS } "An enumeration." -enum SubmissionResultVersion { - ONE - TWO - THREE - OLDEST_SUPPORTED - LATEST +enum ValidationActionType { + ERROR + NO_ACTION + REJECT + WARN } -type AddDataToWorkflow { - subsetId: Int - workflow: Workflow +"An enumeration." +enum ValidationStatus { + ERROR + INFO + REJECT + SUCCESS + WARN } -type SubmitAutoReview { - jobId: String +"An enumeration." +enum WORKFLOWBLUEPRINT_COLUMN_ENUM { + DESCRIPTION + ENABLED + ID + NAME + TYPE + UUID } -type SubmissionResults { - jobId: String +"An enumeration." +enum WORKFLOW_COLUMN_ENUM { + AUTO_REVIEW_ENABLED + CREATED_AT + CREATED_BY + DATASET_ID + EST_HUMAN_TIME_MINS + ID + METRICS_START_DATE + NAME + REVIEWABLE + REVIEW_ENABLED + STATUS + SUBMISSION_RUNNABLE + UNREVIEWABLE_REASON + UNRUNNABLE_REASON + UPDATED_AT + UPDATED_BY + USER_ROLE } -type GenerateUserSnapshotReport { - jobId: String +"An enumeration." +enum WordPredictorStrength { + MODERATE + STRONG + WEAK } -"An enumeration." -enum ReportFormat { - CSV - JSON +"Actions that can be taken on components and links" +enum WorkflowActionOp { + ADD + DELETE } -type GenerateUserChangelogReport { - jobId: String +"An enumeration." +enum WorkflowBlueprintType { + SYSTEM + USER } -input InputProcessor { - processorType: ProcessorType! - args: JSONString +"An enumeration." +enum WorkflowMetricsTag { + PREDICTIONS_COUNT + PREDICTIONS_STP + QUEUES_STAT + SUBMISSION_COMPLETED + SUBMISSION_REVIEW_STAT + SUBMISSION_SUBMITTED } -input QuestionInput { - type: TaskType! - labelsetId: Int - keywords: [String] - text: String! - modelGroupId: Int - targets: [String]! +"An enumeration." +enum WorkflowStatus { + ADDING_DATA + COMPLETE } -type DeleteQuestionnaire { - success: Boolean +"An enumeration." +enum omnipageLanguageCode { + AFR + ALB + AUTO + BAS + BRA + BRE + CAT + CRO + CZH + DAN + DUT + ENG + EST + FIN + FRE + FRI + GAL + GER + GLI + HUN + ICE + IDO + IND + ITA + LAT + LIT + LUX + MLG + MLY + NOR + POL + PRO + ROM + SLK + SLN + SPA + SRL + SUN + SWA + SWE + TAG + TUR + WEL } -type SubmitLabels { - success: Boolean +"An enumeration." +enum readapiLanguageCode { + AFR + ALB + AST + AUTO + AZL + BAS + BEL + BOS + BRA + BRE + BUL + CAT + CEB + CHS + CHT + CRH + CRO + CZH + DAN + DUT + ENG + EST + FIN + FRE + FRI + GAL + GER + GLI + HAT + HUN + ICE + IND + ITA + JAV + JPN + KKC + KKL + KRN + LIT + LUX + MLY + NOR + POL + PRO + ROM + RUS + SCO + SLK + SLN + SPA + SRC + SRL + SWA + SWE + TUR + UZC + VOL + WEL } -input SubmissionLabel { - rowIndex: Int! - target: JSONString - rejected: Boolean = false - deleted: Boolean = false - override: Boolean = false +"An enumeration." +enum readapiTablesV1LanguageCode { + AFR + ALB + ANP + ARA + AST + AUTO + AWA + AZL + BAS + BEL + BFY + BFZ + BGC + BHO + BNS + BOS + BRA + BRAJ + BRE + BRX + BUA + BUL + CAT + CEB + CHS + CHT + CNRC + CNRL + CRH + CRO + CZH + DAN + DHI + DOI + DSB + DUT + ENG + EST + FIN + FRE + FRI + GAG + GAL + GER + GLI + GON + GVR + HAT + HIN + HLB + HNE + HOC + HUN + ICE + IND + ITA + JAV + JNS + JPN + KAA + KFQ + KKC + KKL + KLR + KMJ + KOS + KPY + KRC + KRN + KSH + KUA + KUL + KUM + KYC + LIT + LKT + LTN + LUX + MAR + MLY + MNC + MYV + NEP + NIU + NOG + NOR + OSS + PAS + PER + POL + PRO + PRS + PUN + RAB + ROM + RUS + SAD + SAT + SCK + SCO + SLK + SLN + SMA + SME + SMJ + SMN + SMS + SOA + SPA + SRC + SRL + SRX + SWA + SWE + TAJ + THF + TKL + TUR + TYV + UGA + URD + UZA + UZC + VOL + WEL + XNR + XSR } -type UpdateKeywords { - keywords: [String] +"An enumeration." +enum readapiV2LanguageCode { + AFR + ALB + ANP + ARA + AST + AUTO + AWA + AZL + BAS + BEL + BFY + BFZ + BGC + BHO + BNS + BOS + BRA + BRAJ + BRE + BRX + BUA + BUL + CAT + CEB + CHS + CHT + CNRC + CNRL + CRH + CRO + CZH + DAN + DHI + DOI + DSB + DUT + ENG + EST + FIN + FRE + FRI + GAG + GAL + GER + GLI + GON + GVR + HAT + HIN + HLB + HNE + HOC + HUN + ICE + IND + ITA + JAV + JNS + JPN + KAA + KFQ + KKC + KKL + KLR + KMJ + KOS + KPY + KRC + KRN + KSH + KUA + KUL + KUM + KYC + LIT + LKT + LTN + LUX + MAR + MLY + MNC + MYV + NEP + NIU + NOG + NOR + OSS + PAS + PER + POL + PRO + PRS + PUN + RAB + ROM + RUS + SAD + SAT + SCK + SCO + SLK + SLN + SMA + SME + SMJ + SMN + SMS + SOA + SPA + SRC + SRL + SRX + SWA + SWE + TAJ + THF + TKL + TUR + TYV + UGA + URD + UZA + UZC + VOL + WEL + XNR + XSR +} + +""" +The `Date` scalar type represents a Date +value as specified by +[iso8601](https://en.wikipedia.org/wiki/ISO_8601). +""" +scalar Date + +""" +The `DateTime` scalar type represents a DateTime +value as specified by +[iso8601](https://en.wikipedia.org/wiki/ISO_8601). +""" +scalar DateTime + +""" +ISO-8601 datetime format for JS to parse timezone + +For example: 2020-03-20T01:31:12.467113+00:00 + +This custom scalar should only be used for serializing data out, not as an +input field, so parse_literal and parse_value are not implemented. +""" +scalar DateTimeISO + +""" +The `GenericScalar` scalar type represents a generic +GraphQL scalar value that could be: +String, Boolean, Int, Float, List or Object. +""" +scalar GenericScalar + +"Class to convert to Date for type checking and to String for graphql to pass along" +scalar InputDate + +""" +Allows use of a JSON String for input / output from the GraphQL schema. + +Use of this type is *not recommended* as you lose the benefits of having a defined, static +schema (one of the key benefits of GraphQL). +""" +scalar JSONString + +""" +Recursive filter object that can be used to +produce any filter combination for component blueprints +""" +input ComponentBlueprintFilter { + AND: [ComponentBlueprintFilter] + OR: [ComponentBlueprintFilter] + ands: [ComponentBlueprintFilter] + "Component family to filter for" + componentFamily: ComponentFamily + "Component types to filter for" + componentType: [ComponentType] + "Name to filter by" + name: String + ors: [ComponentBlueprintFilter] + "Tags that blueprints must have superset of" + tags: [BlueprintTag] +} + +"Inputs for a new field specifically for a new component" +input ComponentFieldInput { + "ID of the workflow component to link newly created field to" + componentId: Int! + "The datatype of the field (e.g. 'date', 'price', etc.)" + datatype: String + "Configuration dictionary that controls how the normalized type is displayed in Indico Review UI" + formatConfig: GenericScalar + "Configuration dictionary that controls what inputs are recognized as valid instances of a given type" + inputConfig: GenericScalar + "Denotes whether a single field instance or multiple field instances are expected for an associated field. For instance, an invoice may have multiple 'Line Item Total's (multiple=True), but would be expected to only have a single 'Invoice Date' field (multiple=False)" + multiple: Boolean + "Field name displayed in review UI and the workflow result file" + name: String! + "Controls whether or not a valid instance of a field must be present to submit an review." + required: Boolean + "Configuration that controls which additional validation checks should be run and what actions should be taken in case of their failure." + validationConfig: [ValidationInputConfig] + "Workflow ID" + workflowId: Int! +} + +"Data processing configurations for dataset" +input DataConfigInput { + emailOptions: EmailOptionsInput + ocrOptions: OCROptionsInput +} + +"Filter options for listing datasets" +input DatasetFilter { + AND: [DatasetFilter] + OR: [DatasetFilter] + ands: [DatasetFilter] + "name contains" + name: String + ors: [DatasetFilter] } -type RemoveQuestionnaireUser { - success: Boolean +input DateRangeFilter { + "The starting time to search from" + from: InputDate = null + "The ending time to search until" + to: InputDate = null +} + +"Email options for dataset" +input EmailOptionsInput { + includeSections: EmailSectionOptionsInput = null +} + +"Email sections to include for processing" +input EmailSectionOptionsInput { + attachments: Boolean + body: Boolean + header: Boolean +} + +input ExampleFilter { + "Include or exclude examples that have auto-generated labels (e.g. from GPT-4)" + autolabeled: Boolean + "Examples that do not have labels with these class ids" + excludeClass: [Int] + "Examples for datafile names containing this string" + fileName: String + "Examples for datafiles of these file types" + fileType: [FileType] + "Examples with labels that include these class ids" + includeClass: [Int] + "Examples labeled by these users" + labeler: [Int] + "Include or exclude examples that are partially labeled (e.g. annotated in review)." + partial: Boolean + "Examples currently with these statuses" + status: [ExampleStatus] + "Filter examples based on whether a text string is in the content" + textSearch: String +} + +input ExchangeIntegrationConfigurationInput { + "Filters for inbox ie: is_read, is_from_domain, is_unread, is_not_from_domain, etc. " + filters: JSONString + "Identifier for mailbox folder" + folderId: String + "User's ID" + userId: String +} + +input ExchangeIntegrationCredentialsInput { + "Client id" + clientId: String + "Client secret" + clientSecret: String + "Azure tenant id" + tenantId: String +} + +input FieldInput { + "The datatype of the field (e.g. 'date', 'price', etc.)" + datatype: String + "Configuration dictionary that controls how the normalized type is displayed in Indico Review UI" + formatConfig: GenericScalar + "Configuration dictionary that controls what inputs are recognized as valid instances of a given type" + inputConfig: GenericScalar + "Denotes whether a single field instance or multiple field instances are expected for an associated field. For instance, an invoice may have multiple 'Line Item Total's (multiple=True), but would be expected to only have a single 'Invoice Date' field (multiple=False)" + multiple: Boolean + "Field name displayed in review UI and the workflow result file" + name: String! + "Controls whether or not a valid instance of a field must be present to submit an review." + required: Boolean + "Configuration that controls which additional validation checks should be run and what actions should be taken in case of their failure." + validationConfig: [ValidationInputConfig] } -type CancelModelTraining { - success: Boolean +input FileInput { + """ + JSON string containing the following keys: + - name: filename + - path: indico storage path + - uploadType: must be set to "legacy" + """ + filemeta: JSONString + """ + Name of the file. Note: not the full path. + EG: "sample.pdf" + """ + filename: String } -"An enumeration." -enum FeatureDomainEnum { - STANDARD - STANDARD_V2 - TOPICS - SENTIMENT - FINANCE - EMOTION - ENSEMBLE - FASTTEXT - UNSUPERVISEDSENTIMENT - IMAGE_V2 - IMAGE_V3 - IMAGE_V4 - IMAGE_ENSEMBLE +input LabelInput { + "Denote that this label has been generated automatically and has not been subject to manual review (e.g. has been generated by GPT-3 / GPT-4)" + autolabeled: Boolean + exampleId: Int! + "Override all existing labels for the example with this one. Required True if unrejecting an example. Only available to Analysts+" + override: Boolean + "Denote that not all valid labels present in the document have been captured (e.g. document was processed in review UI). Override must also be set to True to permit flagging a label as partial." + partial: Boolean + "Reject(True) or unreject(False) this example. Unrejection is only available to Analysts+" + rejected: Boolean + targets: [LabelInst] } -"An enumeration." -enum RocAucAveraging { - SIMPLE - WEIGHTED +input LabelInst { + bounds: [SpatialSpanInput] + "id of class name on labelset" + clsId: Int! + spans: [TokenSpanInput] } -"An enumeration." -enum SamplingStrategy { - NO_SAMPLING - RANDOM_OVERSAMPLE +input ModelGroupFilter { + AND: [ModelGroupFilter] + OR: [ModelGroupFilter] + ands: [ModelGroupFilter] + "name contains" + name: String + ors: [ModelGroupFilter] + "model group subset id is" + subsetId: Int + "model group task type is" + taskType: TaskType } -"An enumeration." -enum WordPredictorStrength { - STRONG - MODERATE - WEAK +input NewLabelsetInput { + "Field type, configuration and linking data. If not provided will automatically create fields with corresponding names, or link targets to existing fields with the same name. If an empty list is provided, no fields will be created." + fieldData: [FieldInput] + "Name of the labelset. Defaults to model group name" + name: String + "Number of labelers required" + numLabelersRequired: Int = 1 + "Target name information" + targetNameInputs: [TargetNameInput] + "Labelset task type" + taskType: TaskType } -type DeleteModelGroup { - success: Boolean +input OCROptionsInput { + "Which engine to use for OCR" + ocrEngine: OCREngine = READAPI_V2 + omnipageOptions: OmnipageOcrOptionsInput + readapiOptions: ReadapiOcrOptionsInput + readapiTablesV1Options: ReadapiTablesV1OcrOptionsInput + readapiV2Options: ReadapiV2OcrOptionsInput +} + +input OmnipageOcrOptionsInput { + "Auto rotate" + autoRotate: Boolean + "Return table information for post-processing rules" + cells: Boolean + "Force render" + forceRender: Boolean + "List of languages to use" + languages: [omnipageLanguageCode] + "Native layout" + nativeLayout: Boolean + "Native PDF" + nativePdf: Boolean + "Read table as a single column" + singleColumn: Boolean + "PDF split version" + splitVersion: Int = 2 + "Spreadsheet converter version (supported versions: 1,2,3,4)" + spreadsheetConverterVersion: Int = 3 + "Read table in row or column order" + tableReadOrder: TableReadOrder + "Scale up low resolution images" + upscaleImages: Boolean } -type ModelPredict { - jobId: String +input QuestionInput { + keywords: [String] + labelsetId: Int + modelGroupId: Int + targets: [String]! + "Help text for question" + text: String + type: TaskType! } -type ModelLoad { - status: String +input QuestionnaireFilter { + AND: [QuestionnaireFilter] + OR: [QuestionnaireFilter] + ands: [QuestionnaireFilter] + "name contains" + name: String + ors: [QuestionnaireFilter] } -input FiltersInput { - labels: [String]! - predictionLabelsetId: Int - targetLabelsetId: Int +input QuestionnaireInput { + "Always use Text Labeling UI" + forceTextMode: Boolean = false + "Questionnaire instructions" + instructions: String + "Show predictions at the global level" + showPredictions: Boolean = true + "User IDs to add to the questionnaire" + users: [Int] +} + +input ReadapiOcrOptionsInput { + "Auto rotate" + autoRotate: Boolean + "List of languages to use" + languages: [readapiLanguageCode] + "Read table as a single column" + singleColumn: Boolean + "Spreadsheet converter version (supported versions: 1,2,3,4)" + spreadsheetConverterVersion: Int = 3 + "Scale up low resolution images" + upscaleImages: Boolean +} + +input ReadapiTablesV1OcrOptionsInput { + "Auto rotate" + autoRotate: Boolean + "List of languages to use" + languages: [readapiTablesV1LanguageCode] + "Read table as a single column" + singleColumn: Boolean + "Spreadsheet converter version (supported versions: 1,2,3,4)" + spreadsheetConverterVersion: Int = 3 + "Scale up low resolution images" + upscaleImages: Boolean +} + +input ReadapiV2OcrOptionsInput { + "Auto rotate" + autoRotate: Boolean + "List of languages to use" + languages: [readapiV2LanguageCode] + "Read table as a single column" + singleColumn: Boolean + "Spreadsheet converter version (supported versions: 1,2,3,4)" + spreadsheetConverterVersion: Int = 3 + "Scale up low resolution images" + upscaleImages: Boolean +} + +"Contains the response for a deferred task in a workflow" +input ResumedTaskResponse { + "Response from the task containing outputs and other metadata" + data: JSONString + "Errors during execution to record. Fails the workflow if error is logged" + error: String +} + +input ReviewFilter { + AND: [ReviewFilter] + OR: [ReviewFilter] + ands: [ReviewFilter] + "ID of the user who created the submission" + createdBy: Int + ors: [ReviewFilter] + "Whether submission has been rejected" + rejected: Boolean + "type of review is" + reviewType: ReviewType } -type DeleteFireGroup { - success: Boolean +input ReviewSettingsInput { + "Enable the Auto review queue" + autoReviewQueueEnabled: Boolean + "Add value to Exceptions queue submissions in the workflow" + exceptionsQueueAddValueEnabled: Boolean + "Enable the Exceptions queue" + exceptionsQueueEnabled: Boolean + "Add a rejection reason to review queue submissions in the workflow" + exceptionsQueueRejectionReasonRequired: Boolean + "Enable Submissions List for exceptions queue. Accessible from the Review List page." + exceptionsSubmissionListEnabled: Boolean + "If toggling exceptions queue off, mark existing subs pending admin review as complete. Ignored if toggling exceptions on" + migratePendingAdminReviewToComplete: Boolean = false + "If toggling review off, mark existing submissions waiting for review as complete. Ignored if toggling review on." + migratePendingReviewToComplete: Boolean = false + "If toggling auto review on, mark existing subs pending review as pending auto review. Ignore if toggling off" + migratePendingReviewToPendingAutoReview: Boolean = false + "Add value to Review queue submissions in the workflow" + reviewQueueAddValueEnabled: Boolean + "Enable the Review queue. If disabled, also disables the Exceptions and Auto review queues" + reviewQueueEnabled: Boolean + "Add a rejection reason to Review queue submissions in the workflow" + reviewQueueRejectionReasonRequired: Boolean + "Required number of reviewers per submission in the workflow" + reviewQueueReviewersRequired: Int + "Enable Submissions List for review queue. Accessible from the Review List page." + reviewSubmissionListEnabled: Boolean +} + +input SpatialSpanInput { + bottom: Int! + left: Int! + pageNum: Int! + right: Int! + top: Int! +} + +""" +Filter the submissions by one or more of these attributes. +A filter can be composed of smaller filters using OR/AND +""" +input SubmissionFilter { + AND: [SubmissionFilter] + OR: [SubmissionFilter] + ands: [SubmissionFilter] + "Submissions created during given time range" + createdAt: DateRangeFilter + "Submissions that have had their internal files removed (True) or not (False)" + filesDeleted: Boolean + "submission file(s) are any of these types" + filetype: [FileType!] + "input filename contains" + inputFilename: String + ors: [SubmissionFilter] + "Submission has been marked as having been retrieved" + retrieved: Boolean + "Only show submissions where a review is in progress (or not)" + reviewInProgress: Boolean + "Only show submissions whose completed reviews match these filters" + reviews: ReviewFilter + "submission status is" + status: SubmissionStatus + "Submissions updated during given time range" + updatedAt: DateRangeFilter } -type PDFExtraction { - jobId: String +input SubmissionLabel { + "Must be supplied when rejected is not None and the labelset came from an uploaded csv" + datapointId: Int + "DON’T USE THIS UNTIL 4.14 - OLD, BAD THING" + deleted: Boolean + "Must be manager or analyst to use. Required True if rejected is False (unrejecting an example) or if submitting a label as a user who already submitted a label" + override: Boolean + "True if rejecting an example. Rejecting an example will remove it from all associated labelsets and teach tasks. Set to False to unreject an example and make it available for labeling again." + rejected: Boolean + rowIndex: Int! + "JSON string of target for label" + target: JSONString } -type DocumentExtraction { - jobIds: [String] +input SubmissionLogFilter { + AND: [SubmissionLogFilter] + OR: [SubmissionLogFilter] + ands: [SubmissionLogFilter] + createdAt: DateRangeFilter + "List of submission IDs to filter by" + id: [Int] + ors: [SubmissionLogFilter] + "Status of the submission process to filter by" + status: SubmissionStatus + updatedAt: DateRangeFilter + "List of workflow IDs to filter by" + workflowId: [Int] } -type ActiveFormFields { - jobIds: [String] +input TargetNameInput { + "Name for this TargetName" + name: String! + "Type for this TargetName - if not provided, default is based on Task Type" + targetType: TargetType } -type DeleteDataset { - success: Boolean +input TokenSpanInput { + end: Int! + pageNum: Int! + start: Int! } -type DeleteDatasetUser { - success: Boolean +input UserFilter { + AND: [UserFilter] + OR: [UserFilter] + ands: [UserFilter] + "email contains" + email: String + "name contains" + name: String + ors: [UserFilter] } -"Example:\nmutation generate_new_refresh_token {\n GenerateNewApiRefreshToken{\n refresh_token\n }\n}" -type GenerateNewApiRefreshToken { - refreshToken: String +input UserInput { + email: String! + role: Roles! } -type DeletePreference { - success: Boolean +input UserReportFilter { + AND: [UserReportFilter] + OR: [UserReportFilter] + ands: [UserReportFilter] + ors: [UserReportFilter] + "User email in this list" + userEmail: [String] + "User id in this list" + userId: [Int] } -type Subscription { - datasetCreated: Dataset - datasetUpdated: Dataset - datasetDeleted: Dataset +"Input configuration for a specific validation rule" +input ValidationInputConfig { + "Determines how failure to validate is handled." + onFailure: ValidationActionType + "Name of the validation rule, e.g. 'TYPE_CONVERSION' or 'DATE_RANGE'" + settingName: String + "Setting value for the validation rule, e.g. {'start_date': '2020-01-01', 'end_date': '2021-01-01'}" + settingValue: GenericScalar } -type Component implements ComponentInterface { - id: Int - docbotId: Int - componentAppId: Int - componentType: ComponentType - reviewable: Boolean +input WorkflowBlueprintFilter { + AND: [WorkflowBlueprintFilter] + OR: [WorkflowBlueprintFilter] + ands: [WorkflowBlueprintFilter] + "The workflow blueprint is enabled" + enabled: Boolean + ors: [WorkflowBlueprintFilter] } -type ModelGroupComponent implements ComponentInterface { - id: Int - docbotId: Int - componentAppId: Int - componentType: ComponentType +input WorkflowFilter { + AND: [WorkflowFilter] + OR: [WorkflowFilter] + ands: [WorkflowFilter] + "all new submissions will wait for Auto Review" + autoReviewEnabled: Boolean + "name contains" + name: String + ors: [WorkflowFilter] + "all new submissions will pass through Review" + reviewEnabled: Boolean + "the workflow can use Review for its submissions" reviewable: Boolean - taskType: TaskType - modelType: ModelType -} \ No newline at end of file +} From b99182dc35aa820f57cc8006ec070749283d8ad6 Mon Sep 17 00:00:00 2001 From: Nathanael Shim Date: Tue, 5 Mar 2024 09:46:54 -0800 Subject: [PATCH 2/7] match python sdk changes --- .../Submissions/Models/ISubmission.cs | 2 -- .../Submissions/Models/Submission.cs | 11 ++------- .../Submissions/Models/SubmissionFilter.cs | 23 +------------------ .../Submissions/Submissions.graphql | 4 ---- 4 files changed, 3 insertions(+), 37 deletions(-) diff --git a/IndicoV2.Abstractions/Submissions/Models/ISubmission.cs b/IndicoV2.Abstractions/Submissions/Models/ISubmission.cs index a4a4beab..1d7ee480 100644 --- a/IndicoV2.Abstractions/Submissions/Models/ISubmission.cs +++ b/IndicoV2.Abstractions/Submissions/Models/ISubmission.cs @@ -75,9 +75,7 @@ public interface ISubmission SubmissionRetry[] Retries { get; } - bool? AutoReviewLoaded { get; } - string OcrEngine { get; } Review[] Reviews { get; } diff --git a/IndicoV2.Abstractions/Submissions/Models/Submission.cs b/IndicoV2.Abstractions/Submissions/Models/Submission.cs index 9d457bf6..0d5bb799 100644 --- a/IndicoV2.Abstractions/Submissions/Models/Submission.cs +++ b/IndicoV2.Abstractions/Submissions/Models/Submission.cs @@ -12,7 +12,7 @@ public class SubmissionFile public string FileName { get; set; } - public FileType? FileType { get; set; } + public string? FileType { get; set; } public int? SubmissionId { get; set; } @@ -54,7 +54,6 @@ public class Review public string Notes { get; set; } - public string Changes { get; set; } } public class SubmissionRetry @@ -140,9 +139,7 @@ public class Submission : ISubmission public SubmissionRetry[] Retries { get; set; } - public bool? AutoReviewLoaded { get; set; } - public string OcrEngine { get; set; } public Review[] Reviews { get; set; } @@ -171,7 +168,7 @@ public SubmissionSs(IListSubmissions_Submissions_Submissions submission) Id = inputFile.Id, FilePath = inputFile.Filepath, FileName = inputFile.Filename, - FileType = (FileType)inputFile.Filetype, + FileType = inputFile.Filetype.ToString(), SubmissionId = inputFile.SubmissionId, FileSize = inputFile.FileSize, NumPages = inputFile.NumPages @@ -192,7 +189,6 @@ public SubmissionSs(IListSubmissions_Submissions_Submissions submission) Rejected = _ssSubmission.AutoReview.Rejected, ReviewType = (ReviewType)_ssSubmission.AutoReview.ReviewType, Notes = _ssSubmission.AutoReview.Notes, - Changes = _ssSubmission.AutoReview.Changes } : new Review() { }; Retries = _ssSubmission.Retries.Select(submissionRetry => new SubmissionRetry { @@ -202,8 +198,6 @@ public SubmissionSs(IListSubmissions_Submissions_Submissions submission) PreviousStatus = (SubmissionStatus)submissionRetry.PreviousStatus, RetryErrors = submissionRetry.RetryErrors }).ToArray(); - AutoReviewLoaded = _ssSubmission.AutoReviewLoaded; - OcrEngine = _ssSubmission.OcrEngine; Reviews = _ssSubmission.Reviews.Select(review => new Review { Id = review.Id, @@ -215,7 +209,6 @@ public SubmissionSs(IListSubmissions_Submissions_Submissions submission) Rejected = review.Rejected, ReviewType = (ReviewType)review.ReviewType, Notes = review.Notes, - Changes = review.Changes }).ToArray(); ReviewInProgress = _ssSubmission.ReviewInProgress; } diff --git a/IndicoV2.Abstractions/Submissions/Models/SubmissionFilter.cs b/IndicoV2.Abstractions/Submissions/Models/SubmissionFilter.cs index 23da99ae..670cd244 100644 --- a/IndicoV2.Abstractions/Submissions/Models/SubmissionFilter.cs +++ b/IndicoV2.Abstractions/Submissions/Models/SubmissionFilter.cs @@ -1,26 +1,5 @@ namespace IndicoV2.Submissions.Models { - public enum FileType - { - Csv, - Doc, - Docx, - Eml, - Excel, - Jpg, - Msg, - Pdf, - Png, - Ppt, - Pptx, - Rtf, - Tiff, - Txt, - Unknown, - Xls, - Xlsx - } - public enum ReviewType { Admin, @@ -43,7 +22,7 @@ public class DateRangeFilter : IFilter public class SubmissionFilter : IFilter { - public FileType[] FileType { get; set; } + public string[] FileType { get; set; } public string InputFilename { get; set; } public SubmissionStatus? Status { get; set; } public bool? Retrieved { get; set; } diff --git a/IndicoV2.StrawberryShake/Submissions/Submissions.graphql b/IndicoV2.StrawberryShake/Submissions/Submissions.graphql index e0dfed15..49d94ded 100644 --- a/IndicoV2.StrawberryShake/Submissions/Submissions.graphql +++ b/IndicoV2.StrawberryShake/Submissions/Submissions.graphql @@ -90,7 +90,6 @@ query ListSubmissions( rejected reviewType notes - changes } retries { id @@ -99,8 +98,6 @@ query ListSubmissions( previousStatus retryErrors } - AutoReviewLoaded - ocrEngine reviews { id submissionId @@ -111,7 +108,6 @@ query ListSubmissions( rejected reviewType notes - changes } reviewInProgress } From a0ac0f5d2890ddde09c0a9417cb70a042fa5d38a Mon Sep 17 00:00:00 2001 From: Nathanael Shim Date: Tue, 5 Mar 2024 10:03:20 -0800 Subject: [PATCH 3/7] add xml comments --- .../Submissions/Models/FilterConverter.cs | 13 +- .../Submissions/Models/Submission.cs | 116 +++++++++++++++++- 2 files changed, 121 insertions(+), 8 deletions(-) diff --git a/IndicoV2.Abstractions/Submissions/Models/FilterConverter.cs b/IndicoV2.Abstractions/Submissions/Models/FilterConverter.cs index 76618b4c..502179ca 100644 --- a/IndicoV2.Abstractions/Submissions/Models/FilterConverter.cs +++ b/IndicoV2.Abstractions/Submissions/Models/FilterConverter.cs @@ -18,22 +18,25 @@ public static ssSubmissionFilter ConvertToSs(this IFilter filter) { return new ssSubmissionFilter() { - Filetype = submissionFilter.FileType.Select(x => (StrawberryShake.FileType)x).ToList(), + Filetype = submissionFilter.FileType.Select(x => (StrawberryShake.FileType)Enum.Parse(typeof(StrawberryShake.FileType), x)).ToList(), InputFilename = submissionFilter.InputFilename, Retrieved = submissionFilter.Retrieved, Status = (StrawberryShake.SubmissionStatus?)submissionFilter.Status, - Reviews = new ReviewFilter() { + Reviews = new ReviewFilter() + { Rejected = submissionFilter.Reviews.Rejected, CreatedBy = submissionFilter.Reviews.CreatedBy, - ReviewType = (StrawberryShake.ReviewType) submissionFilter.Reviews.ReviewType + ReviewType = (StrawberryShake.ReviewType)submissionFilter.Reviews.ReviewType }, ReviewInProgress = submissionFilter.ReviewInProgress, FilesDeleted = submissionFilter.FilesDeleted, - CreatedAt = new StrawberryShake.DateRangeFilter() { + CreatedAt = new StrawberryShake.DateRangeFilter() + { From = submissionFilter.CreatedAt.From, To = submissionFilter.CreatedAt.To, }, - UpdatedAt = new StrawberryShake.DateRangeFilter() { + UpdatedAt = new StrawberryShake.DateRangeFilter() + { From = submissionFilter.UpdatedAt.From, To = submissionFilter.UpdatedAt.To } diff --git a/IndicoV2.Abstractions/Submissions/Models/Submission.cs b/IndicoV2.Abstractions/Submissions/Models/Submission.cs index 0d5bb799..78c00e82 100644 --- a/IndicoV2.Abstractions/Submissions/Models/Submission.cs +++ b/IndicoV2.Abstractions/Submissions/Models/Submission.cs @@ -6,66 +6,145 @@ namespace IndicoV2.Submissions.Models { public class SubmissionFile { + /// + /// Unique Id of this file. + /// public int? Id { get; set; } + /// + /// Local URL to stored input. + /// public string FilePath { get; set; } + /// + /// Name of original file. + /// public string FileName { get; set; } - public string? FileType { get; set; } + /// + /// Type of file. + /// + public string FileType { get; set; } + /// + /// Id of the submission this file is associated with. + /// public int? SubmissionId { get; set; } + /// + /// Size of file in bytes. + /// public int? FileSize { get; set; } + /// + /// Number of pages in file. + /// public int? NumPages { get; set; } } public class SubmissionOutput { + /// + /// Unique Id of this output. + /// public int? Id { get; set; } + /// + /// Local URL to stored input. + /// public string FilePath { get; set; } + /// + /// Id of the submission this file is associated with. + /// public int? SubmissionId { get; set; } + /// + /// Id of the workflow component that made this file. + /// public int? ComponentId { get; set; } + /// + /// Datetime the output file was created. + /// public DateTimeOffset? CreatedAt { get; set; } } public class Review { + /// + /// Id of this review. + /// public int? Id { get; set; } + /// + /// Id of the submission this review is associated with. + /// public int? SubmissionId { get; set; } + /// + /// When this user first opened the file. See startedAt as well. + /// public string CreatedAt { get; set; } + /// + /// Reviewer id. + /// public int? CreatedBy { get; set; } + /// + /// When this review was started. + /// Differs from createdAt because a reviewer may restart their review at any time. + /// public string StartedAt { get; set; } + /// + /// When this review was completed by the reviewer. + /// public string CompletedAt { get; set; } + /// + /// Flag for whether the file was rejected (True) or not (False). + /// public bool? Rejected { get; set; } + /// + /// Type of review. + /// public ReviewType? ReviewType { get; set; } + /// + /// Reviewer notes. + /// public string Notes { get; set; } } public class SubmissionRetry { + /// + /// Unique Id of the submission retry. + /// public int? Id { get; set; } + /// + /// Unique Id of the associated submission. + /// public int? SubmissionId { get; set; } + /// + /// Errors from previous submission. + /// public string PreviousErrors { get; set; } + /// + /// Status of submission before it was retried. + /// public SubmissionStatus? PreviousStatus { get; set; } + /// + /// Errors that ocurred during the retrying of this submission. + /// public string RetryErrors { get; set; } } @@ -91,14 +170,29 @@ public class Submission : ISubmission /// public SubmissionStatus Status { get; set; } + /// + /// Datetime the submission was created. + /// public DateTimeOffset? CreatedAt { get; set; } + /// + /// Datetime the submission was updated. + /// public DateTimeOffset? UpdatedAt { get; set; } + /// + /// Id of the user who created the submission. + /// public int? CreatedBy { get; set; } + /// + /// Id of the user who updated the submission. + /// public int? UpdatedBy { get; set; } + /// + /// Datetime the submission reached a completed state. + /// public DateTimeOffset? CompletedAt { get; set; } /// @@ -111,6 +205,9 @@ public class Submission : ISubmission /// public bool? FilesDeleted { get; set; } + /// + /// Submission input files. + /// public SubmissionFile[] InputFiles { get; set; } /// @@ -128,6 +225,9 @@ public class Submission : ISubmission /// public string ResultFile { get; set; } + /// + /// Output files of the submission. + /// public SubmissionOutput[] OutputFiles { get; set; } /// @@ -135,14 +235,24 @@ public class Submission : ISubmission /// public bool Retrieved { get; set; } + /// + /// Latest auto review for submission. + /// public Review AutoReview { get; set; } + /// + /// List of retries for submission. + /// public SubmissionRetry[] Retries { get; set; } - - + /// + /// Completed review of this submission, without changes. + /// public Review[] Reviews { get; set; } + /// + /// True if the submission is being actively reviewed. + /// public bool? ReviewInProgress { get; set; } } From bd7eddd7a395d8fa11f408f4f506979d1b74c119 Mon Sep 17 00:00:00 2001 From: Nathanael Shim Date: Wed, 6 Mar 2024 09:41:01 -0800 Subject: [PATCH 4/7] store changes --- .../Submissions/Models/FilterConverter.cs | 14 ++-- .../Submissions/Models/SubmissionFilter.cs | 11 ++- .../Storage/StorageClientTests.cs | 4 +- .../Submissions/SubmissionClientTests.cs | 68 +++++++++++++++++++ .../IndicoV2.StrawberryShake.csproj | 4 +- IndicoV2/IndicoV2.csproj | 2 +- IndicoV2/Submissions/SubmissionsClient.cs | 1 + 7 files changed, 91 insertions(+), 13 deletions(-) diff --git a/IndicoV2.Abstractions/Submissions/Models/FilterConverter.cs b/IndicoV2.Abstractions/Submissions/Models/FilterConverter.cs index 502179ca..3a803d92 100644 --- a/IndicoV2.Abstractions/Submissions/Models/FilterConverter.cs +++ b/IndicoV2.Abstractions/Submissions/Models/FilterConverter.cs @@ -18,28 +18,28 @@ public static ssSubmissionFilter ConvertToSs(this IFilter filter) { return new ssSubmissionFilter() { - Filetype = submissionFilter.FileType.Select(x => (StrawberryShake.FileType)Enum.Parse(typeof(StrawberryShake.FileType), x)).ToList(), + Filetype = submissionFilter.FileType != null ? submissionFilter.FileType.Select(x => (FileType)Enum.Parse(typeof(FileType), x)).ToList() : new List(), InputFilename = submissionFilter.InputFilename, Retrieved = submissionFilter.Retrieved, Status = (StrawberryShake.SubmissionStatus?)submissionFilter.Status, - Reviews = new ReviewFilter() + Reviews = submissionFilter.Reviews != null ? new ReviewFilter() { Rejected = submissionFilter.Reviews.Rejected, CreatedBy = submissionFilter.Reviews.CreatedBy, ReviewType = (StrawberryShake.ReviewType)submissionFilter.Reviews.ReviewType - }, + } : null, ReviewInProgress = submissionFilter.ReviewInProgress, FilesDeleted = submissionFilter.FilesDeleted, - CreatedAt = new StrawberryShake.DateRangeFilter() + CreatedAt = submissionFilter.CreatedAt != null ? new StrawberryShake.DateRangeFilter() { From = submissionFilter.CreatedAt.From, To = submissionFilter.CreatedAt.To, - }, - UpdatedAt = new StrawberryShake.DateRangeFilter() + } : null, + UpdatedAt = submissionFilter.UpdatedAt != null ? new StrawberryShake.DateRangeFilter() { From = submissionFilter.UpdatedAt.From, To = submissionFilter.UpdatedAt.To - } + } : null }; } else if (filter is AndFilter andfilter) diff --git a/IndicoV2.Abstractions/Submissions/Models/SubmissionFilter.cs b/IndicoV2.Abstractions/Submissions/Models/SubmissionFilter.cs index 670cd244..6ed7f84f 100644 --- a/IndicoV2.Abstractions/Submissions/Models/SubmissionFilter.cs +++ b/IndicoV2.Abstractions/Submissions/Models/SubmissionFilter.cs @@ -1,4 +1,6 @@ -namespace IndicoV2.Submissions.Models +using Newtonsoft.Json; + +namespace IndicoV2.Submissions.Models { public enum ReviewType { @@ -24,12 +26,19 @@ public class SubmissionFilter : IFilter { public string[] FileType { get; set; } public string InputFilename { get; set; } + public SubmissionStatus? Status { get; set; } + public bool? Retrieved { get; set; } + public SubmissionReviewFilter Reviews { get; set; } + public bool? ReviewInProgress { get; set; } + public bool? FilesDeleted { get; set; } + public DateRangeFilter CreatedAt { get; set; } + public DateRangeFilter UpdatedAt { get; set; } } } diff --git a/IndicoV2.IntegrationTests/Storage/StorageClientTests.cs b/IndicoV2.IntegrationTests/Storage/StorageClientTests.cs index d2fcc301..2598a5ec 100644 --- a/IndicoV2.IntegrationTests/Storage/StorageClientTests.cs +++ b/IndicoV2.IntegrationTests/Storage/StorageClientTests.cs @@ -29,10 +29,10 @@ public async Task UploadAsync_ShouldReturnUploadedFileInfo() { // Arrange var filePath = _dataHelper.Files().GetSampleFilePath(); - + // Act var uploadedFiles = (await _storageClient.UploadAsync(new[] {filePath}, default)).ToArray(); - + // Assert uploadedFiles.Should().HaveCount(1); var uploadedFile = uploadedFiles.Single(); diff --git a/IndicoV2.IntegrationTests/Submissions/SubmissionClientTests.cs b/IndicoV2.IntegrationTests/Submissions/SubmissionClientTests.cs index dcf99308..b67da18a 100644 --- a/IndicoV2.IntegrationTests/Submissions/SubmissionClientTests.cs +++ b/IndicoV2.IntegrationTests/Submissions/SubmissionClientTests.cs @@ -163,6 +163,74 @@ public async Task ListAsync_ShouldFetchSubmissionsWithCursor() } + [Test] + public async Task ListAsync_SubmissionFilter_ShouldFetchSubmissions() + { + // Arrange + var listData = await _dataHelper.Submissions().ListAnyAsync(_workflowId); + + var filters = new OrFilter + { + Or = new List + { + new SubmissionFilter + { + InputFilename = "", + Status = SubmissionStatus.COMPLETE, + Retrieved = false + }, + new SubmissionFilter + { + InputFilename = "", + Status = SubmissionStatus.FAILED, + Retrieved = false + } + } + }; + + // Act + var submissions = await _submissionsClient.ListAsync(new List { listData.submissionId }, new List { listData.workflowId }, filters, 0, 1000); + + submissions.Should().NotBeNull(); + submissions.PageInfo.Should().NotBeNull(); + submissions.Data.Should().NotBeNull(); + var submission = submissions.Data.First(); + + // Assert + submissions.Data.Should().HaveCountGreaterThan(0); + submission.Id.Should().BeGreaterThan(0); + submission.Status.Should().BeOfType(); + } + + + [Test] + public async Task ListSubmissions_SubmissionFilterReviewInProgress_ShouldFetchSubmissions() + { + // Arrange + var listData = await _dataHelper.Submissions().ListAnyAsync(_workflowId); + + var filters = new SubmissionFilter + { + InputFilename = "", + Status = SubmissionStatus.COMPLETE, + Retrieved = false + }; + + // Act + var submissions = await _submissionsClient.ListAsync(new List { listData.submissionId }, new List { listData.workflowId }, filters, 0, 1000); + + submissions.Should().NotBeNull(); + submissions.PageInfo.Should().NotBeNull(); + submissions.Data.Should().NotBeNull(); + var submission = submissions.Data.First(); + + // Assert + submissions.Data.Should().HaveCountGreaterThan(0); + submission.Id.Should().BeGreaterThan(0); + submission.Status.Should().BeOfType(); + } + + [Test] public async Task GenerateSubmissionResultAsync_ShouldReturnJob() { diff --git a/IndicoV2.StrawberryShake/IndicoV2.StrawberryShake.csproj b/IndicoV2.StrawberryShake/IndicoV2.StrawberryShake.csproj index 2a1c242a..aface62c 100644 --- a/IndicoV2.StrawberryShake/IndicoV2.StrawberryShake.csproj +++ b/IndicoV2.StrawberryShake/IndicoV2.StrawberryShake.csproj @@ -14,8 +14,8 @@ - - + + diff --git a/IndicoV2/IndicoV2.csproj b/IndicoV2/IndicoV2.csproj index 5d304801..e9be6986 100644 --- a/IndicoV2/IndicoV2.csproj +++ b/IndicoV2/IndicoV2.csproj @@ -50,7 +50,7 @@ - + diff --git a/IndicoV2/Submissions/SubmissionsClient.cs b/IndicoV2/Submissions/SubmissionsClient.cs index a878469c..4124cc46 100644 --- a/IndicoV2/Submissions/SubmissionsClient.cs +++ b/IndicoV2/Submissions/SubmissionsClient.cs @@ -102,6 +102,7 @@ public async Task>> ListAsync(IEnumerable)submissionIds.Select(x => (int?)x).ToList().AsReadOnly(); var readonlyWorkflowIds = (IReadOnlyList)workflowIds.Select(x => (int?)x).ToList().AsReadOnly(); + Console.WriteLine("asdfsafasffddfdfdfddfd"); var result = await _strawberryShakeClient.Submissions().List(readonlyIds, readonlyWorkflowIds, ssFilters, limit, after, cancellationToken); return new HasCursor>() From c550cf7de2a051e212c686d42771620cae8d66fd Mon Sep 17 00:00:00 2001 From: Nathanael Shim Date: Wed, 6 Mar 2024 13:20:30 -0800 Subject: [PATCH 5/7] resolve failing tests --- .../Submissions/Models/FilterConverter.cs | 54 ++++++++++++++----- 1 file changed, 40 insertions(+), 14 deletions(-) diff --git a/IndicoV2.Abstractions/Submissions/Models/FilterConverter.cs b/IndicoV2.Abstractions/Submissions/Models/FilterConverter.cs index 3a803d92..bb0aa8b7 100644 --- a/IndicoV2.Abstractions/Submissions/Models/FilterConverter.cs +++ b/IndicoV2.Abstractions/Submissions/Models/FilterConverter.cs @@ -16,31 +16,57 @@ public static ssSubmissionFilter ConvertToSs(this IFilter filter) { if (filter is v2SubmissionFilter submissionFilter) { - return new ssSubmissionFilter() + var ssFilter = new ssSubmissionFilter(); + if (submissionFilter.FileType != null) + { + ssFilter.Filetype = submissionFilter.FileType.Select(x => (FileType)Enum.Parse(typeof(FileType), x)).ToList(); + } + if (submissionFilter.InputFilename != null) + { + ssFilter.InputFilename = submissionFilter.InputFilename; + } + if (submissionFilter.Retrieved != null) + { + ssFilter.Retrieved = submissionFilter.Retrieved; + } + if (submissionFilter.Status != null) + { + ssFilter.Status = (StrawberryShake.SubmissionStatus?)submissionFilter.Status; + } + if (submissionFilter.Reviews != null) { - Filetype = submissionFilter.FileType != null ? submissionFilter.FileType.Select(x => (FileType)Enum.Parse(typeof(FileType), x)).ToList() : new List(), - InputFilename = submissionFilter.InputFilename, - Retrieved = submissionFilter.Retrieved, - Status = (StrawberryShake.SubmissionStatus?)submissionFilter.Status, - Reviews = submissionFilter.Reviews != null ? new ReviewFilter() + ssFilter.Reviews = new ReviewFilter() { Rejected = submissionFilter.Reviews.Rejected, CreatedBy = submissionFilter.Reviews.CreatedBy, ReviewType = (StrawberryShake.ReviewType)submissionFilter.Reviews.ReviewType - } : null, - ReviewInProgress = submissionFilter.ReviewInProgress, - FilesDeleted = submissionFilter.FilesDeleted, - CreatedAt = submissionFilter.CreatedAt != null ? new StrawberryShake.DateRangeFilter() + }; + } + if (submissionFilter.ReviewInProgress != null) + { + ssFilter.ReviewInProgress = submissionFilter.ReviewInProgress; + } + if (submissionFilter.FilesDeleted != null) + { + ssFilter.FilesDeleted = submissionFilter.FilesDeleted; + } + if (submissionFilter.CreatedAt != null) + { + ssFilter.CreatedAt = new StrawberryShake.DateRangeFilter() { From = submissionFilter.CreatedAt.From, To = submissionFilter.CreatedAt.To, - } : null, - UpdatedAt = submissionFilter.UpdatedAt != null ? new StrawberryShake.DateRangeFilter() + }; + } + if (submissionFilter.UpdatedAt != null) + { + ssFilter.UpdatedAt = new StrawberryShake.DateRangeFilter() { From = submissionFilter.UpdatedAt.From, To = submissionFilter.UpdatedAt.To - } : null - }; + }; + } + return ssFilter; } else if (filter is AndFilter andfilter) { From a58443c6f789f99dee1778e5e0a855cb46338281 Mon Sep 17 00:00:00 2001 From: Nathanael Shim Date: Wed, 6 Mar 2024 13:42:38 -0800 Subject: [PATCH 6/7] add tests --- Examples/SubmissionFilterExample/Program.cs | 2 - .../Submissions/Models/ISubmission.cs | 46 +++++++++++-- .../Submissions/SubmissionClientTests.cs | 68 +++++++++++++++++-- IndicoV2/Submissions/SubmissionsClient.cs | 1 - 4 files changed, 101 insertions(+), 16 deletions(-) diff --git a/Examples/SubmissionFilterExample/Program.cs b/Examples/SubmissionFilterExample/Program.cs index 0bce3119..340fa489 100644 --- a/Examples/SubmissionFilterExample/Program.cs +++ b/Examples/SubmissionFilterExample/Program.cs @@ -22,13 +22,11 @@ public static async Task Main() { new SubmissionFilter { - InputFilename = "", Status = SubmissionStatus.COMPLETE, Retrieved = false }, new SubmissionFilter { - InputFilename = "", Status = SubmissionStatus.FAILED, Retrieved = false } diff --git a/IndicoV2.Abstractions/Submissions/Models/ISubmission.cs b/IndicoV2.Abstractions/Submissions/Models/ISubmission.cs index 1d7ee480..1bec47e9 100644 --- a/IndicoV2.Abstractions/Submissions/Models/ISubmission.cs +++ b/IndicoV2.Abstractions/Submissions/Models/ISubmission.cs @@ -1,6 +1,5 @@  using System; -using IndicoV2.CommonModels.Pagination; namespace IndicoV2.Submissions.Models { @@ -29,41 +28,64 @@ public interface ISubmission /// SubmissionStatus Status { get; } + /// + /// Datetime the submission was created. + /// DateTimeOffset? CreatedAt { get; } + /// + /// Datetime the submission was updated. + /// DateTimeOffset? UpdatedAt { get; } + /// + /// Id of the user who created the submission. + /// int? CreatedBy { get; } + /// + /// Id of the user who updated the submission. + /// int? UpdatedBy { get; } + /// + /// Datetime the submission reached a completed state. + /// DateTimeOffset? CompletedAt { get; } - /// /// Submission errors. /// string Errors { get; } + /// + /// Submission files have been deleted from file store. + /// bool? FilesDeleted { get; } + /// + /// List of submission input files. + /// SubmissionFile[] InputFiles { get; } /// - /// Submission input file. + /// Local URL to first stored input. /// string InputFile { get; } /// - /// Submission name of input file. + /// Original name of first file. /// string InputFilename { get; } /// - /// Submission result file. + /// Local URL to most recently stored output. /// string ResultFile { get; } + /// + /// List of submission output files. + /// SubmissionOutput[] OutputFiles { get; } /// @@ -71,14 +93,24 @@ public interface ISubmission /// bool Retrieved { get; } + /// + /// Latest auto review for submission. + /// Review AutoReview { get; } + /// + /// List of submission retries. + /// SubmissionRetry[] Retries { get; } - - + /// + /// Completed reviews of this submission, without changes. + /// Review[] Reviews { get; } + /// + /// True if the submission is being actively reviewed. + /// bool? ReviewInProgress { get; } } diff --git a/IndicoV2.IntegrationTests/Submissions/SubmissionClientTests.cs b/IndicoV2.IntegrationTests/Submissions/SubmissionClientTests.cs index b67da18a..fa2b4b72 100644 --- a/IndicoV2.IntegrationTests/Submissions/SubmissionClientTests.cs +++ b/IndicoV2.IntegrationTests/Submissions/SubmissionClientTests.cs @@ -11,6 +11,7 @@ using IndicoV2.IntegrationTests.Utils.Configs; using NUnit.Framework; using Unity; +using System; namespace IndicoV2.IntegrationTests.Submissions { @@ -175,13 +176,11 @@ public async Task ListAsync_SubmissionFilter_ShouldFetchSubmissions() { new SubmissionFilter { - InputFilename = "", Status = SubmissionStatus.COMPLETE, Retrieved = false }, new SubmissionFilter { - InputFilename = "", Status = SubmissionStatus.FAILED, Retrieved = false } @@ -211,13 +210,40 @@ public async Task ListSubmissions_SubmissionFilterReviewInProgress_ShouldFetchSu var filters = new SubmissionFilter { - InputFilename = "", - Status = SubmissionStatus.COMPLETE, - Retrieved = false + ReviewInProgress = null }; // Act - var submissions = await _submissionsClient.ListAsync(new List { listData.submissionId }, new List { listData.workflowId }, filters, 0, 1000); + var submissions = await _submissionsClient.ListAsync(new List { }, new List { listData.workflowId }, filters, 0, 10); + + submissions.Should().NotBeNull(); + submissions.PageInfo.Should().NotBeNull(); + submissions.Data.Should().NotBeNull(); + var submission = submissions.Data.First(); + + // Assert + submissions.Data.Should().HaveCountGreaterThan(0); + submission.Id.Should().BeGreaterThan(0); + submission.Status.Should().BeOfType(); + } + + + [Test] + public async Task ListSubmissions_SubmissionFilterCreatedAt_ShouldFetchSubmissions() + { + // Arrange + var listData = await _dataHelper.Submissions().ListAnyAsync(_workflowId); + + var filters = new SubmissionFilter + { + CreatedAt = new DateRangeFilter(){ + From = "2022-01-01", + To = DateTime.Now.ToString("yyyy-MM-dd") + } + }; + + // Act + var submissions = await _submissionsClient.ListAsync(new List { }, new List { listData.workflowId }, filters, 0, 10); submissions.Should().NotBeNull(); submissions.PageInfo.Should().NotBeNull(); @@ -231,6 +257,36 @@ public async Task ListSubmissions_SubmissionFilterReviewInProgress_ShouldFetchSu } + [Test] + public async Task ListSubmissions_SubmissionFilterUpdatedAt_ShouldFetchSubmissions() + { + // Arrange + var listData = await _dataHelper.Submissions().ListAnyAsync(_workflowId); + + var filters = new SubmissionFilter + { + UpdatedAt = new DateRangeFilter(){ + From = "2022-01-01", + To = DateTime.Now.ToString("yyyy-MM-dd") + } + }; + + // Act + var submissions = await _submissionsClient.ListAsync(new List { }, new List { listData.workflowId }, filters, 0, 10); + + submissions.Should().NotBeNull(); + submissions.PageInfo.Should().NotBeNull(); + submissions.Data.Should().NotBeNull(); + var submission = submissions.Data.First(); + + // Assert + submissions.Data.Should().HaveCountGreaterThan(0); + submission.Id.Should().BeGreaterThan(0); + submission.Status.Should().BeOfType(); + } + + + [Test] public async Task GenerateSubmissionResultAsync_ShouldReturnJob() { diff --git a/IndicoV2/Submissions/SubmissionsClient.cs b/IndicoV2/Submissions/SubmissionsClient.cs index 4124cc46..a878469c 100644 --- a/IndicoV2/Submissions/SubmissionsClient.cs +++ b/IndicoV2/Submissions/SubmissionsClient.cs @@ -102,7 +102,6 @@ public async Task>> ListAsync(IEnumerable)submissionIds.Select(x => (int?)x).ToList().AsReadOnly(); var readonlyWorkflowIds = (IReadOnlyList)workflowIds.Select(x => (int?)x).ToList().AsReadOnly(); - Console.WriteLine("asdfsafasffddfdfdfddfd"); var result = await _strawberryShakeClient.Submissions().List(readonlyIds, readonlyWorkflowIds, ssFilters, limit, after, cancellationToken); return new HasCursor>() From d5e768d052f8536e37ded4e9a14f8e01d761ea97 Mon Sep 17 00:00:00 2001 From: Nathanael Shim Date: Wed, 6 Mar 2024 14:00:00 -0800 Subject: [PATCH 7/7] address comments; edit examples --- Examples/SubmissionFilterExample/Program.cs | 74 +++++++++++++++++-- .../Submissions/Models/FilterConverter.cs | 11 +-- 2 files changed, 72 insertions(+), 13 deletions(-) diff --git a/Examples/SubmissionFilterExample/Program.cs b/Examples/SubmissionFilterExample/Program.cs index 340fa489..05e4404d 100644 --- a/Examples/SubmissionFilterExample/Program.cs +++ b/Examples/SubmissionFilterExample/Program.cs @@ -16,30 +16,88 @@ private static string GetToken() => public static async Task Main() { - var filters = new OrFilter //https://developer.indicodata.ai/docs/filter-submissions + var client = new IndicoClient(GetToken(), new Uri("https://try.indico.io")); + + var submissionClient = client.Submissions(); + var submissionIds = new List() { }; + var workflowIds = new List() { 3106 }; + + // Example 1 + // List all submissions that are complete or failed + var orFilter = new OrFilter { Or = new List { new SubmissionFilter { Status = SubmissionStatus.COMPLETE, - Retrieved = false }, new SubmissionFilter { Status = SubmissionStatus.FAILED, - Retrieved = false } } }; - var client = new IndicoClient(GetToken(), new Uri("https://try.indico.io")); + var submissions = await submissionClient.ListAsync(submissionIds, workflowIds, orFilter, limit: 10); + Console.WriteLine(submissions); - var submissionClient = client.Submissions(); - var submissionIds = new List() { }; - var workflowIds = new List() { 3106 }; + // Example 2 + // List all submissions that are COMPLETE and FAILED + var andFilter = new AndFilter + { + And = new List + { + new SubmissionFilter + { + Status = SubmissionStatus.COMPLETE, + }, + new SubmissionFilter + { + Status = SubmissionStatus.FAILED, + } + } + }; + + submissions = await submissionClient.ListAsync(submissionIds, workflowIds, andFilter, limit: 10); + Console.WriteLine(submissions); + + // Example 3 + // List all submissions that are retrieved and have a filename that contains 'property' + var subFilter = new SubmissionFilter + { + Retrieved = true, + InputFilename = "property" + }; + + submissions = await submissionClient.ListAsync(submissionIds, workflowIds, subFilter, limit: 10); + Console.WriteLine(submissions); + + // Example 4 + // List all submissions that are created and updated within a certain date range + var dateRangeFilter = new DateRangeFilter() + { + From = "2022-01-01", + To = DateTime.Now.ToString("yyyy-MM-dd") + }; + subFilter = new SubmissionFilter + { + CreatedAt = dateRangeFilter, + UpdatedAt = dateRangeFilter + }; + + submissions = await submissionClient.ListAsync(submissionIds, workflowIds, subFilter, limit: 10); + Console.WriteLine(submissions); + + // Example 5 + // List all submissions that are not in progress of being reviewed and are completed + subFilter = new SubmissionFilter + { + Status = SubmissionStatus.COMPLETE, + ReviewInProgress = false + }; - var submissions = await submissionClient.ListAsync(submissionIds, workflowIds, filters, limit: 1000); + submissions = await submissionClient.ListAsync(submissionIds, workflowIds, subFilter, limit: 10); Console.WriteLine(submissions); } } diff --git a/IndicoV2.Abstractions/Submissions/Models/FilterConverter.cs b/IndicoV2.Abstractions/Submissions/Models/FilterConverter.cs index bb0aa8b7..4c93c548 100644 --- a/IndicoV2.Abstractions/Submissions/Models/FilterConverter.cs +++ b/IndicoV2.Abstractions/Submissions/Models/FilterConverter.cs @@ -16,20 +16,21 @@ public static ssSubmissionFilter ConvertToSs(this IFilter filter) { if (filter is v2SubmissionFilter submissionFilter) { + // Note: you have to not reference or set a field at all for StrawberryShake Code Generator to decide to not serialize it var ssFilter = new ssSubmissionFilter(); if (submissionFilter.FileType != null) { ssFilter.Filetype = submissionFilter.FileType.Select(x => (FileType)Enum.Parse(typeof(FileType), x)).ToList(); } - if (submissionFilter.InputFilename != null) + if (string.IsNullOrEmpty(submissionFilter.InputFilename)) { ssFilter.InputFilename = submissionFilter.InputFilename; } - if (submissionFilter.Retrieved != null) + if (submissionFilter.Retrieved.HasValue) { ssFilter.Retrieved = submissionFilter.Retrieved; } - if (submissionFilter.Status != null) + if (submissionFilter.Status.HasValue) { ssFilter.Status = (StrawberryShake.SubmissionStatus?)submissionFilter.Status; } @@ -42,11 +43,11 @@ public static ssSubmissionFilter ConvertToSs(this IFilter filter) ReviewType = (StrawberryShake.ReviewType)submissionFilter.Reviews.ReviewType }; } - if (submissionFilter.ReviewInProgress != null) + if (submissionFilter.ReviewInProgress.HasValue) { ssFilter.ReviewInProgress = submissionFilter.ReviewInProgress; } - if (submissionFilter.FilesDeleted != null) + if (submissionFilter.FilesDeleted.HasValue) { ssFilter.FilesDeleted = submissionFilter.FilesDeleted; }