feat: query endpoint (#1)

Co-authored-by: Sebastian <[email protected]>

.env

@@ -1,2 +1,5 @@
 SENTRY_DSN=
 PORT=
+DATABASE_URL=https://dev.aam-digital.net/db
+QUERY_URL=http://localhost:3002
+SCHEMA_CONFIG_ID=_design/sqlite:config

README.md

@@ -1,5 +1,16 @@
-# Deployer Backend
+# Query Backend
 
-This server allows to automatically deploy applications on the server.
+This service allows to run SQL queries on the database.
+In particular, this service allows users with limited permissions to see reports of aggregated statistics across all data (e.g. a supervisor could analyse reports without having access to possibly confidential details of participants or notes).
 
+## Usage
+See the [ndb-setup repo](https://github.com/Aam-Digital/ndb-setup) for full deployment instructions.
 
+To use this you need a running [CouchDB](https://docs.couchdb.org/en/stable/) and [structured query server (SQS)](https://neighbourhood.ie/products-and-services/structured-query-server).
+
+The following variables might need to be configured in the `.env` file:
+- `DATABASE_URL` URL of the `CouchDB` or [replication backend](https://github.com/Aam-Digital/replication-backend)
+- `QUERY_URL` URL of the SQS
+- `SCHEMA_CONFIG_ID` database ID of the document which holds the SQS schema (default `_design/sqlite:config`)
+- `PORT` where the app should listen (default 3000)
+- `SENTRY_DSN` for remote logging

build/Dockerfile

@@ -34,6 +34,8 @@ COPY --from=builder /app/dist ./dist
 # (optional) The sentry DSN in order to send the error messages to sentry
 ENV SENTRY_DSN=""
 ENV PORT=""
+ENV DATABASE_URL=""
+ENV QUERY_URL=""
 
 CMD ["node", "dist/main"]
 

nest-cli.json

@@ -2,7 +2,6 @@
   "collection": "@nestjs/schematics",
   "sourceRoot": "src",
   "compilerOptions": {
-    "plugins": ["@nestjs/swagger"],
-    "assets": ["assets/*"]
+    "plugins": ["@nestjs/swagger"]
   }
 }

src/app.controller.spec.ts

@@ -1,20 +1,44 @@
 import { AppController } from './app.controller';
 import { Test, TestingModule } from '@nestjs/testing';
-import { of } from 'rxjs';
+import { of, throwError } from 'rxjs';
 import { HttpService } from '@nestjs/axios';
-import { ConfigModule } from '@nestjs/config';
+import { BadRequestException, HttpException, HttpStatus } from '@nestjs/common';
+import { SqlReport } from './sql-report';
+import { ConfigService } from '@nestjs/config';
+import { QueryBody } from './query-body.dto';
 
 describe('AppController', () => {
   let controller: AppController;
-  let mockHttp: { post: jest.Mock };
+  let mockHttp: { post: jest.Mock; get: jest.Mock };
+  const dbUrl = 'database:3000';
+  const queryUrl = 'query:3000';
+  const schemaConfigId = '_design/sqlite:config';
 
   beforeEach(async () => {
     mockHttp = {
       post: jest.fn().mockReturnValue(of({ data: undefined })),
+      get: jest.fn().mockReturnValue(of({ data: undefined })),
+    };
+    const mockConfigService = {
+      get: (key) => {
+        switch (key) {
+          case 'DATABASE_URL':
+            return dbUrl;
+          case 'QUERY_URL':
+            return queryUrl;
+          case 'SCHEMA_CONFIG_ID':
+            return schemaConfigId;
+          default:
+            throw Error('missing mock value for ' + key);
+        }
+      },
     };
     const module: TestingModule = await Test.createTestingModule({
-      imports: [ConfigModule],
-      providers: [AppController, { provide: HttpService, useValue: mockHttp }],
+      providers: [
+        AppController,
+        { provide: HttpService, useValue: mockHttp },
+        { provide: ConfigService, useValue: mockConfigService },
+      ],
     }).compile();
 
     controller = module.get(AppController);
@@ -23,4 +47,133 @@ describe('AppController', () => {
   it('should create', () => {
     expect(controller).toBeDefined();
   });
+
+  it('should forward report query to SQS and return result', (done) => {
+    const report: SqlReport = {
+      mode: 'sql',
+      aggregationDefinitions: ['SELECT * FROM someTable'],
+    };
+    mockHttp.get.mockReturnValue(of({ data: report }));
+    const queryResult = [{ some: 'data' }];
+    mockHttp.post.mockReturnValue(of({ data: queryResult }));
+
+    controller
+      .queryData('ReportConfig:some-id', 'app', 'valid token')
+      .subscribe((res) => {
+        expect(mockHttp.get).toHaveBeenCalledWith(
+          `${dbUrl}/app/ReportConfig:some-id`,
+          { headers: { Authorization: 'valid token' } },
+        );
+        expect(mockHttp.post).toHaveBeenCalledWith(
+          `${queryUrl}/app/${schemaConfigId}`,
+          { query: report.aggregationDefinitions[0] },
+        );
+        expect(res).toEqual(queryResult);
+
+        done();
+      });
+  });
+
+  it('should add dates as args to query request', (done) => {
+    const report: SqlReport = {
+      mode: 'sql',
+      aggregationDefinitions: [
+        'SELECT * FROM Note WHERE e.date BETWEEN ? AND  ?',
+      ],
+    };
+    mockHttp.get.mockReturnValue(of({ data: report }));
+    const body: QueryBody = { from: '2023-01-01', to: '2024-01-01' };
+
+    controller
+      .queryData('ReportConfig:some-id', 'app', 'valid token', body)
+      .subscribe(() => {
+        expect(mockHttp.post).toHaveBeenCalledWith(
+          `${queryUrl}/app/${schemaConfigId}`,
+          {
+            query: report.aggregationDefinitions[0],
+            args: [body.from, body.to],
+          },
+        );
+        done();
+      });
+  });
+
+  it('should concatenate the result of multiple SELECT queries', (done) => {
+    const firstResult = [{ value: 'first' }, { value: 'second' }];
+    const secondResult = [{ value: 'third' }];
+    const report: SqlReport = {
+      mode: 'sql',
+      aggregationDefinitions: ['SELECT * FROM Child', 'SELECT * FROM School'],
+    };
+    mockHttp.get.mockReturnValue(of({ data: report }));
+    mockHttp.post
+      .mockReturnValueOnce(of({ data: firstResult }))
+      .mockReturnValueOnce(of({ data: secondResult }));
+
+    controller
+      .queryData('ReportConfig:some-id', 'app', 'valid token')
+      .subscribe((res) => {
+        expect(mockHttp.post).toHaveBeenCalledWith(
+          `${queryUrl}/app/${schemaConfigId}`,
+          { query: report.aggregationDefinitions[0] },
+        );
+        expect(mockHttp.post).toHaveBeenCalledWith(
+          `${queryUrl}/app/${schemaConfigId}`,
+          { query: report.aggregationDefinitions[1] },
+        );
+        expect(res).toEqual([...firstResult, ...secondResult]);
+
+        done();
+      });
+  });
+
+  it('should throw error if user is not permitted to request report', (done) => {
+    mockHttp.get.mockReturnValue(
+      throwError(() => ({
+        response: { data: 'Unauthorized', status: 401 },
+      })),
+    );
+    controller
+      .queryData('ReportConfig:some-id', 'app', 'invalid token')
+      .subscribe({
+        error: (err: HttpException) => {
+          expect(err.getStatus()).toBe(HttpStatus.UNAUTHORIZED);
+          done();
+        },
+      });
+  });
+
+  it('should throw error trying to query a non-sql report', (done) => {
+    const report: SqlReport = {
+      mode: 'exporting' as any,
+      aggregationDefinitions: undefined,
+    };
+    mockHttp.get.mockReturnValue(of({ data: report }));
+
+    controller
+      .queryData('ReportConfig:some-id', 'app', 'valid token')
+      .subscribe({
+        error: (err) => {
+          expect(err).toBeInstanceOf(BadRequestException);
+          done();
+        },
+      });
+  });
+
+  it('should throw sql query is not defined', (done) => {
+    const report: SqlReport = {
+      mode: 'sql',
+      aggregationDefinitions: undefined,
+    };
+    mockHttp.get.mockReturnValue(of({ data: report }));
+
+    controller
+      .queryData('ReportConfig:some-id', 'app', 'valid token')
+      .subscribe({
+        error: (err) => {
+          expect(err).toBeInstanceOf(BadRequestException);
+          done();
+        },
+      });
+  });
 });

src/app.controller.ts

@@ -1,13 +1,97 @@
-import { Controller, } from '@nestjs/common';
+import {
+  BadRequestException,
+  Body,
+  Controller,
+  Headers,
+  HttpException,
+  Param,
+  Post,
+} from '@nestjs/common';
 import { HttpService } from '@nestjs/axios';
 import { ConfigService } from '@nestjs/config';
+import { ApiHeader, ApiOperation, ApiParam } from '@nestjs/swagger';
+import { catchError, concat, map, mergeMap, toArray } from 'rxjs';
+import { SqlReport } from './sql-report';
+import { QueryBody } from './query-body.dto';
 
-@Controller()
+@Controller('report')
 export class AppController {
-
+  private dbUrl = this.configService.get('DATABASE_URL');
+  private queryUrl = this.configService.get('QUERY_URL');
+  private schemaDocId = this.configService.get('SCHEMA_CONFIG_ID');
   constructor(
     private http: HttpService,
     private configService: ConfigService,
+  ) {}
+
+  // TODO also support cookie auth? Not really required with Keycloak
+  @ApiOperation({
+    description: `Get the results for the report with the given ID. User needs 'read' access for the requested report entity.`,
+  })
+  @ApiParam({ name: 'id', description: '(full) ID of the report entity' })
+  @ApiParam({ name: 'db', example: 'app', description: 'name of database' })
+  @ApiHeader({
+    name: 'Authorization',
+    required: false,
+    description: 'request needs to be authenticated',
+  })
+  @Post(':db/:id')
+  queryData(
+    @Param('id') reportId: string,
+    @Param('db') db: string,
+    @Headers('Authorization') token: string,
+    @Body() body?: QueryBody,
   ) {
+    return this.http
+      .get<SqlReport>(`${this.dbUrl}/${db}/${reportId}`, {
+        headers: { Authorization: token },
+      })
+      .pipe(
+        mergeMap(({ data }) => this.executeReport(data, db, body)),
+        catchError((err) => {
+          throw err.response?.data
+            ? new HttpException(err.response.data, err.response.status)
+            : err;
+        }),
+      );
+  }
+
+  private executeReport(report: SqlReport, db: string, args?: QueryBody) {
+    if (report.mode !== 'sql') {
+      throw new BadRequestException('Not an SQL report');
+    }
+    if (!report.aggregationDefinitions) {
+      throw new BadRequestException('Report query not configured');
+    }
+
+    // execute all requests in sequence
+    return concat(
+      ...report.aggregationDefinitions.map((query) =>
+        this.getQueryResult(query, args, db),
+      ),
+    ).pipe(
+      // combine results of each request
+      toArray(),
+      map((res) => [].concat(...res)),
+    );
   }
+
+  private getQueryResult(query: string, args: QueryBody, db: string) {
+    const data: SqsRequest = { query: query };
+    if (args?.from && args?.to) {
+      data.args = [args.from, args.to];
+    }
+    return this.http
+      .post<any[]>(`${this.queryUrl}/${db}/${this.schemaDocId}`, data)
+      .pipe(map(({ data }) => data));
+  }
+}
+
+/**
+ * Request body as required by the SQS service. See SQS docs for more info.
+ * {@link https://neighbourhood.ie/products-and-services/structured-query-server}
+ */
+interface SqsRequest {
+  query: string;
+  args?: any[];
 }

src/main.ts

@@ -11,9 +11,13 @@ async function bootstrap() {
     .setTitle(process.env.npm_package_name)
     .setDescription(process.env.npm_package_description)
     .setVersion(process.env.npm_package_version)
+    .addBearerAuth(undefined, 'BearerAuth')
+    .addSecurityRequirements('BearerAuth')
     .build();
   const document = SwaggerModule.createDocument(app, config);
-  SwaggerModule.setup('api', app, document);
+  SwaggerModule.setup('api', app, document, {
+    swaggerOptions: { persistAuthorization: true },
+  });
 
   // Logging everything through sentry
   app.useLogger(SentryService.SentryServiceInstance());

src/query-body.dto.ts

@@ -0,0 +1,9 @@
+/**
+ * The dates can be used in the SQL SELECT statements with a "?"
+ * "from" will replace the first "?"
+ * "to" will replace the second "?"
+ */
+export class QueryBody {
+  from: string;
+  to: string;
+}

src/sql-report.ts

@@ -0,0 +1,8 @@
+/**
+ * The report entity needs to have the following format in order to work.
+ * This aligns with the same interface in {@link https://github.com/Aam-Digital/ndb-core}
+ */
+export interface SqlReport {
+  mode: 'sql';
+  aggregationDefinitions: string[];
+}