diff --git a/app/scripts/backend/backend.ts b/app/scripts/backend/backend.ts index 0e5e954f..e219d8a4 100644 --- a/app/scripts/backend/backend.ts +++ b/app/scripts/backend/backend.ts @@ -1,27 +1,13 @@ /** @format */ import _ from "lodash" -import angular, { IDeferred, IHttpService, IPromise, IQService } from "angular" import { getAuthorizationHeader } from "@/components/auth/auth" import { KorpResponse, WithinParameters } from "@/backend/types" import { SavedSearch } from "@/local-storage" import settings from "@/settings" -import { httpConfAddMethod, httpConfAddMethodAngular } from "@/util" +import { httpConfAddMethodFetch } from "@/util" import { KorpStatsResponse, normalizeStatsData } from "@/backend/stats-proxy" import { MapResult, parseMapData } from "@/map_services" import { KorpQueryResponse } from "@/backend/kwic-proxy" -import "@/backend/lexicons" - -export type BackendService = { - requestCompare: (cmpObj1: SavedSearch, cmpObj2: SavedSearch, reduce: string[]) => IPromise - requestMapData: ( - cqp: string, - cqpExprs: Record, - within: WithinParameters, - attribute: MapAttribute, - relative: boolean - ) => IPromise - getDataForReadingMode: (inputCorpus: string, textId: string) => IPromise | void> -} type KorpLoglikeResponse = { /** Log-likelihood average. */ @@ -33,13 +19,17 @@ type KorpLoglikeResponse = { /** Absolute frequency for the values in set 2. */ set2: Record } + export type CompareResult = [CompareTables, number, SavedSearch, SavedSearch, string[]] + export type CompareTables = { positive: CompareItem[]; negative: CompareItem[] } + type CompareItemRaw = { value: string loglike: number abs: number } + export type CompareItem = { /** Value of given attribute without probability suffixes */ key: string @@ -59,158 +49,143 @@ export type MapRequestResult = { data: MapResult[] attribute: MapAttribute } + type MapAttribute = { label: string; corpora: string[] } -angular.module("korpApp").factory("backend", [ - "$http", - "$q", - "lexicons", - ($http: IHttpService, $q: IQService): BackendService => ({ - requestCompare(cmpObj1, cmpObj2, reduce) { - reduce = _.map(reduce, (item) => item.replace(/^_\./, "")) - let cl = settings.corpusListing - // remove all corpora which do not include all the "reduce"-attributes - const corpora1 = cmpObj1.corpora.filter((corpus) => cl.corpusHasAttrs(corpus, reduce)) - const corpora2 = cmpObj2.corpora.filter((corpus) => cl.corpusHasAttrs(corpus, reduce)) - - const attrs = { ...cl.getCurrentAttributes(), ...cl.getStructAttrs() } - const split = reduce.filter((r) => (attrs[r] && attrs[r].type) === "set").join(",") - - const rankedReduce = _.filter(reduce, (item) => cl.getCurrentAttributes(cl.getReduceLang())[item]?.ranked) - const top = rankedReduce.map((item) => item + ":1").join(",") - - const def: IDeferred = $q.defer() - const params = { - group_by: reduce.join(","), - set1_corpus: corpora1.join(",").toUpperCase(), - set1_cqp: cmpObj1.cqp, - set2_corpus: corpora2.join(",").toUpperCase(), - set2_cqp: cmpObj2.cqp, - max: 50, - split, - top, - } - - const conf = httpConfAddMethodAngular({ - url: settings["korp_backend_url"] + "/loglike", - method: "GET", - params, - headers: getAuthorizationHeader(), - }) - - const xhr = $http>(conf) - - xhr.then(function (response) { - const { data } = response - - if ("ERROR" in data) { - def.reject() - return - } - - const objs: CompareItemRaw[] = _.map(data.loglike, (value, key) => ({ - value: key, - loglike: value, - abs: value > 0 ? data.set2[key] : data.set1[key], - })) - - const tables = _.groupBy(objs, (obj) => (obj.loglike > 0 ? "positive" : "negative")) - - let max = 0 - const groupAndSum = function (table: CompareItemRaw[]) { - // Merge items that are different only by probability suffix ":" - const groups = _.groupBy(table, (obj) => obj.value.replace(/(:.+?)(\/|$| )/g, "$2")) - const res = _.map(groups, (items, key): CompareItem => { - // Add up similar items. - const tokenLists = key.split("/").map((tokens) => tokens.split(" ")) - const loglike = _.sumBy(items, "loglike") - const abs = _.sumBy(items, "abs") - const elems = items.map((item) => item.value) - max = Math.max(max, Math.abs(loglike)) - return { key, loglike, abs, elems, tokenLists } - }) - return res - } - const positive = groupAndSum(tables.positive) - const negative = groupAndSum(tables.negative) - - return def.resolve([{ positive, negative }, max, cmpObj1, cmpObj2, reduce]) - }) - - return def.promise - }, - - requestMapData(cqp, cqpExprs, within, attribute, relative) { - const cqpSubExprs = {} - _.map(_.keys(cqpExprs), (subCqp, idx) => (cqpSubExprs[`subcqp${idx}`] = subCqp)) - - const params = { - group_by_struct: attribute.label, - cqp, - corpus: attribute.corpora.join(","), - incremental: true, - split: attribute.label, - relative_to_struct: relative ? attribute.label : undefined, - } - _.extend(params, settings.corpusListing.getWithinParameters()) - - _.extend(params, cqpSubExprs) - - const conf = httpConfAddMethod({ - url: settings["korp_backend_url"] + "/count", - method: "GET", - params, - headers: getAuthorizationHeader(), - }) - - return $http(conf).then( - function ({ data }) { - const normalizedData = normalizeStatsData(data) as any // TODO Type correctly - let result = parseMapData(normalizedData, cqp, cqpExprs) - return { corpora: attribute.corpora, cqp, within, data: result, attribute } - }, - (err) => { - console.log("err", err) - } - ) - }, - - getDataForReadingMode(inputCorpus, textId) { - const corpus = inputCorpus.toUpperCase() - const corpusSettings = settings.corpusListing.get(inputCorpus) - - // TODO: is this good enough? - const show = _.keys(corpusSettings.attributes) - const showStruct = _.keys(corpusSettings["struct_attributes"]) - - const params = { - corpus: corpus, - cqp: `[_.text__id = "${textId}" & lbound(text)]`, - context: corpus + ":1 text", - // _head and _tail are needed for all corpora, so that Korp will know what whitespace to use - // For sentence_id, we should find a more general solution, but here is one Språkbanken - // corpus who needs sentence_id in order to map the selected sentence in the KWIC to - // a sentence in the reading mode text. - show: show.join(",") + ",sentence_id,_head,_tail", - show_struct: showStruct.join(","), - within: corpus + ":text", - start: 0, - end: 0, - } - - const conf = httpConfAddMethod({ - url: settings["korp_backend_url"] + "/query", - method: "GET", - params, - headers: getAuthorizationHeader(), - }) - - return $http>(conf).then( - (response) => response.data, - (err) => { - console.log("err", err) - } - ) - }, - }), -]) +async function korpRequest = {}, P extends Record = {}>( + endpoint: string, + params: P +): Promise> { + const { url, request } = httpConfAddMethodFetch(settings.korp_backend_url + "/" + endpoint, params) + request.headers = { ...request.headers, ...getAuthorizationHeader() } + const response = await fetch(url, request) + return (await response.json()) as KorpResponse +} + +/** Note: since this is using native Promise, we must use it with something like $q or $scope.$apply for AngularJS to react when they resolve. */ +export async function requestCompare( + cmpObj1: SavedSearch, + cmpObj2: SavedSearch, + reduce: string[] +): Promise { + reduce = _.map(reduce, (item) => item.replace(/^_\./, "")) + let cl = settings.corpusListing + // remove all corpora which do not include all the "reduce"-attributes + const corpora1 = cmpObj1.corpora.filter((corpus) => cl.corpusHasAttrs(corpus, reduce)) + const corpora2 = cmpObj2.corpora.filter((corpus) => cl.corpusHasAttrs(corpus, reduce)) + + const attrs = { ...cl.getCurrentAttributes(), ...cl.getStructAttrs() } + const split = reduce.filter((r) => (attrs[r] && attrs[r].type) === "set").join(",") + + const rankedReduce = _.filter(reduce, (item) => cl.getCurrentAttributes(cl.getReduceLang())[item]?.ranked) + const top = rankedReduce.map((item) => item + ":1").join(",") + + const params = { + group_by: reduce.join(","), + set1_corpus: corpora1.join(",").toUpperCase(), + set1_cqp: cmpObj1.cqp, + set2_corpus: corpora2.join(",").toUpperCase(), + set2_cqp: cmpObj2.cqp, + max: "50", + split, + top, + } + + const data = await korpRequest("loglike", params) + + if ("ERROR" in data) { + // TODO Create a KorpBackendError which could be displayed nicely + throw new Error(data.ERROR.value) + } + + const objs: CompareItemRaw[] = _.map(data.loglike, (value, key) => ({ + value: key, + loglike: value, + abs: value > 0 ? data.set2[key] : data.set1[key], + })) + + const tables = _.groupBy(objs, (obj) => (obj.loglike > 0 ? "positive" : "negative")) + + let max = 0 + const groupAndSum = function (table: CompareItemRaw[]) { + // Merge items that are different only by probability suffix ":" + const groups = _.groupBy(table, (obj) => obj.value.replace(/(:.+?)(\/|$| )/g, "$2")) + const res = _.map(groups, (items, key): CompareItem => { + // Add up similar items. + const tokenLists = key.split("/").map((tokens) => tokens.split(" ")) + const loglike = _.sumBy(items, "loglike") + const abs = _.sumBy(items, "abs") + const elems = items.map((item) => item.value) + max = Math.max(max, Math.abs(loglike)) + return { key, loglike, abs, elems, tokenLists } + }) + return res + } + const positive = groupAndSum(tables.positive) + const negative = groupAndSum(tables.negative) + + return [{ positive, negative }, max, cmpObj1, cmpObj2, reduce] +} + +export async function requestMapData( + cqp: string, + cqpExprs: Record, + within: WithinParameters, + attribute: MapAttribute, + relative: boolean +): Promise { + const cqpSubExprs = {} + _.map(_.keys(cqpExprs), (subCqp, idx) => (cqpSubExprs[`subcqp${idx}`] = subCqp)) + + const params = { + group_by_struct: attribute.label, + cqp, + corpus: attribute.corpora.join(","), + incremental: true, + split: attribute.label, + relative_to_struct: relative ? attribute.label : undefined, + } + _.extend(params, settings.corpusListing.getWithinParameters()) + + _.extend(params, cqpSubExprs) + + const data = await korpRequest("count", params) + + if ("ERROR" in data) { + // TODO Create a KorpBackendError which could be displayed nicely + throw new Error(data.ERROR.value) + } + + const normalizedData = normalizeStatsData(data) as any // TODO Type correctly + let result = parseMapData(normalizedData, cqp, cqpExprs) + return { corpora: attribute.corpora, cqp, within, data: result, attribute } +} + +export async function getDataForReadingMode( + inputCorpus: string, + textId: string +): Promise> { + const corpus = inputCorpus.toUpperCase() + const corpusSettings = settings.corpusListing.get(inputCorpus) + + // TODO: is this good enough? + const show = _.keys(corpusSettings.attributes) + const showStruct = _.keys(corpusSettings["struct_attributes"]) + + const params = { + corpus: corpus, + cqp: `[_.text__id = "${textId}" & lbound(text)]`, + context: corpus + ":1 text", + // _head and _tail are needed for all corpora, so that Korp will know what whitespace to use + // For sentence_id, we should find a more general solution, but here is one Språkbanken + // corpus who needs sentence_id in order to map the selected sentence in the KWIC to + // a sentence in the reading mode text. + show: show.join(",") + ",sentence_id,_head,_tail", + show_struct: showStruct.join(","), + within: corpus + ":text", + start: 0, + end: 0, + } + + return korpRequest("query", params) +} diff --git a/app/scripts/components/compare-search.js b/app/scripts/components/compare-search.js index 2bfddd4b..aea376e7 100644 --- a/app/scripts/components/compare-search.js +++ b/app/scripts/components/compare-search.js @@ -5,6 +5,7 @@ import settings from "@/settings" import "@/backend/backend" import "@/services/compare-searches" import { html, valfilter } from "@/util" +import { requestCompare } from "@/backend/backend" angular.module("korpApp").component("compareSearch", { template: html` @@ -44,10 +45,10 @@ angular.module("korpApp").component("compareSearch", { `, controller: [ - "backend", + "$q", "$rootScope", "compareSearches", - function (backend, $rootScope, compareSearches) { + function ($q, $rootScope, compareSearches) { const $ctrl = this $ctrl.valfilter = valfilter @@ -77,7 +78,7 @@ angular.module("korpApp").component("compareSearch", { $ctrl.reduce = "word" $ctrl.sendCompare = () => - $rootScope.compareTabs.push(backend.requestCompare($ctrl.cmp1, $ctrl.cmp2, [$ctrl.reduce])) + $rootScope.compareTabs.push($q.resolve(requestCompare($ctrl.cmp1, $ctrl.cmp2, [$ctrl.reduce]))) $ctrl.deleteCompares = () => compareSearches.flush() }, diff --git a/app/scripts/components/statistics.js b/app/scripts/components/statistics.js index 03a35c1e..87accf0f 100644 --- a/app/scripts/components/statistics.js +++ b/app/scripts/components/statistics.js @@ -172,8 +172,7 @@ angular.module("korpApp").component("statistics", { "$scope", "$uibModal", "searches", - "backend", - function ($rootScope, $scope, $uibModal, searches, backend) { + function ($rootScope, $scope, $uibModal, searches) { const $ctrl = this $ctrl.noRowsError = false @@ -412,7 +411,7 @@ angular.module("korpApp").component("statistics", { const selectedAttribute = selectedAttributes[0] const within = settings.corpusListing.subsetFactory(selectedAttribute.corpora).getWithinParameters() - const request = backend.requestMapData(cqpExpr, cqpExprs, within, selectedAttribute, $ctrl.mapRelative) + const request = requestMapData(cqpExpr, cqpExprs, within, selectedAttribute, $ctrl.mapRelative) $rootScope.mapTabs.push(request) } diff --git a/app/scripts/controllers/text_reader_controller.ts b/app/scripts/controllers/text_reader_controller.ts index 7e61849c..09f1207d 100644 --- a/app/scripts/controllers/text_reader_controller.ts +++ b/app/scripts/controllers/text_reader_controller.ts @@ -7,10 +7,10 @@ import "@/backend/backend" import "@/components/readingmode" import { RootScope, TextTab } from "@/root-scope.types" import { CorpusTransformed } from "@/settings/config-transformed.types" -import { BackendService } from "@/backend/backend" import { kebabize } from "@/util" import { ApiKwic, Token } from "@/backend/kwic-proxy" import { TabHashScope } from "@/directives/tab-hash" +import { getDataForReadingMode } from "@/backend/backend" type TextReaderControllerScope = TabHashScope & { loading: boolean @@ -48,13 +48,11 @@ type TokenTreeLeaf = { } angular.module("korpApp").directive("textReaderCtrl", [ - "$timeout", - ($timeout) => ({ + () => ({ controller: [ "$scope", "$rootScope", - "backend", - ($scope: TextReaderControllerScope, $rootScope: RootScope, backend: BackendService) => { + ($scope: TextReaderControllerScope, $rootScope: RootScope) => { $scope.loading = true $scope.newDynamicTab() @@ -67,14 +65,16 @@ angular.module("korpApp").directive("textReaderCtrl", [ const corpus = $scope.inData.corpus $scope.corpusObj = settings.corpora[corpus] const textId = $scope.inData.sentenceData["text__id"] - backend.getDataForReadingMode(corpus, textId).then(function (data) { + getDataForReadingMode(corpus, textId).then(function (data) { if (!data || "ERROR" in data) { - $timeout(() => ($scope.loading = false)) + $scope.$apply(($scope: TextReaderControllerScope) => ($scope.loading = false)) return } const document = prepareData(data.kwic[0], $scope.corpusObj) - $scope.data = { corpus, document, sentenceData: $scope.inData.sentenceData } - $timeout(() => ($scope.loading = false)) + $scope.$apply(($scope: TextReaderControllerScope) => { + $scope.data = { corpus, document, sentenceData: $scope.inData.sentenceData } + $scope.loading = false + }) }) $scope.onentry = function () { diff --git a/app/scripts/util.ts b/app/scripts/util.ts index 17399a99..05736756 100644 --- a/app/scripts/util.ts +++ b/app/scripts/util.ts @@ -451,7 +451,7 @@ export function httpConfAddMethodAngular -): { url: string; request?: RequestInit } { +): { url: string; request: RequestInit } { if (calcUrlLength(url, params) > settings.backendURLMaxLength) { const body = new FormData() for (const key in params) { @@ -459,7 +459,7 @@ export function httpConfAddMethodFetch( } return { url, request: { method: "POST", body } } } else { - return { url: url + "?" + new URLSearchParams(params) } + return { url: url + "?" + new URLSearchParams(params), request: {} } } }