From d443c910bf340a6b17aacb1f27d24fae32444fa6 Mon Sep 17 00:00:00 2001 From: Olivia Guyot Date: Mon, 22 Jul 2024 14:36:15 +0200 Subject: [PATCH 1/7] fix(gn4-api): adjust content-type for records api to XML Also regenerates API client This fixes an error when trying to save a record with GN 4.2.5+ --- .../gn4/src/openapi/api/records.api.service.ts | 6 +----- libs/data-access/gn4/src/spec.yaml | 8 -------- 2 files changed, 1 insertion(+), 13 deletions(-) diff --git a/libs/data-access/gn4/src/openapi/api/records.api.service.ts b/libs/data-access/gn4/src/openapi/api/records.api.service.ts index 6884ce91a7..1867765db7 100644 --- a/libs/data-access/gn4/src/openapi/api/records.api.service.ts +++ b/libs/data-access/gn4/src/openapi/api/records.api.service.ts @@ -7201,11 +7201,7 @@ export class RecordsApiService { } // to determine the Content-Type header - const consumes: string[] = [ - 'application/xml', - 'application/json', - 'application/x-www-form-urlencoded', - ] + const consumes: string[] = ['application/xml'] const httpContentTypeSelected: string | undefined = this.configuration.selectHeaderContentType(consumes) if (httpContentTypeSelected !== undefined) { diff --git a/libs/data-access/gn4/src/spec.yaml b/libs/data-access/gn4/src/spec.yaml index ee451ac7a1..bfe1eb0b59 100644 --- a/libs/data-access/gn4/src/spec.yaml +++ b/libs/data-access/gn4/src/spec.yaml @@ -1832,14 +1832,6 @@ paths: schema: type: string description: XML fragment. - application/json: - schema: - type: string - description: XML fragment. - application/x-www-form-urlencoded: - schema: - type: string - description: XML fragment. responses: default: description: default response From c0329f42cce22892c7d532d88ad09c75f37fc513 Mon Sep 17 00:00:00 2001 From: Olivia Guyot Date: Mon, 22 Jul 2024 17:26:56 +0200 Subject: [PATCH 2/7] e2e(me): write basic test for editor form --- apps/metadata-editor-e2e/src/e2e/edit.cy.ts | 60 +++++++++++++++++++ .../top-toolbar/top-toolbar.component.html | 2 + tools/e2e/commands.ts | 13 ++++ 3 files changed, 75 insertions(+) create mode 100644 apps/metadata-editor-e2e/src/e2e/edit.cy.ts diff --git a/apps/metadata-editor-e2e/src/e2e/edit.cy.ts b/apps/metadata-editor-e2e/src/e2e/edit.cy.ts new file mode 100644 index 0000000000..ed249ca863 --- /dev/null +++ b/apps/metadata-editor-e2e/src/e2e/edit.cy.ts @@ -0,0 +1,60 @@ +describe('editor form', () => { + beforeEach(() => { + cy.login('admin', 'admin', false) + + // Alpine convention record + cy.visit('/edit/8698bf0b-fceb-4f0f-989b-111e7c4af0a4') + + cy.clearRecordDrafts() + + // aliases + cy.get('gn-ui-form-field[ng-reflect-model=abstract] textarea').as( + 'abstractField' + ) + cy.get('@abstractField').invoke('val').as('abstractFieldInitialValue') + cy.get('[data-cy=save-status]') + .invoke('attr', 'data-cy-value') + .as('saveStatus') + }) + + it('form shows correctly', () => { + cy.get('gn-ui-record-form').should('be.visible') + cy.get('gn-ui-record-form gn-ui-form-field').should('have.length.gt', 0) + cy.get('@abstractField') + .invoke('val') + .should('contain', 'Perimeter der Alpenkonvention in der Schweiz.') + cy.get('@saveStatus').should('eq', 'record_up_to_date') + cy.screenshot({ capture: 'fullPage' }) + }) + + it('draft record is kept', () => { + cy.get('@abstractField').clear() + cy.get('@abstractField').type('modified abstract') + // eslint-disable-next-line cypress/no-unnecessary-waiting + cy.wait(1000) // waiting for draft saving to kick in + cy.reload() + cy.get('@abstractField').invoke('val').should('eq', 'modified abstract') + cy.get('@saveStatus').should('eq', 'draft_changes_pending') + + cy.clearRecordDrafts() + + cy.get('@saveStatus').should('eq', 'record_up_to_date') + cy.get('@abstractField') + .invoke('val') + .should('contain', 'Perimeter der Alpenkonvention in der Schweiz.') + }) + + it('saving record works', () => { + cy.get('@abstractField').clear() + cy.get('@abstractField').type('modified abstract before saving') + cy.get('md-editor-publish-button').click() + cy.get('@saveStatus').should('eq', 'record_up_to_date') + + // restore abstract + cy.get('@abstractField').clear() + cy.get('@abstractField').then(function (field) { + cy.wrap(field).type(this.abstractFieldInitialValue) + }) + cy.get('md-editor-publish-button').click() + }) +}) diff --git a/apps/metadata-editor/src/app/edit/components/top-toolbar/top-toolbar.component.html b/apps/metadata-editor/src/app/edit/components/top-toolbar/top-toolbar.component.html index 89cd41f9d4..b1f5cec581 100644 --- a/apps/metadata-editor/src/app/edit/components/top-toolbar/top-toolbar.component.html +++ b/apps/metadata-editor/src/app/edit/components/top-toolbar/top-toolbar.component.html @@ -13,6 +13,8 @@
diff --git a/tools/e2e/commands.ts b/tools/e2e/commands.ts index 0c06e446e3..736b225890 100644 --- a/tools/e2e/commands.ts +++ b/tools/e2e/commands.ts @@ -15,6 +15,7 @@ declare namespace Cypress { login(username?: string, password?: string, redirect?: boolean): void signOut(): void clearFavorites(): void + clearRecordDrafts(): void // interaction with gn-ui-dropdown-selector openDropdown(): Chainable> @@ -139,6 +140,18 @@ Cypress.Commands.add( } ) +Cypress.Commands.add('clearRecordDrafts', () => { + cy.window().then((window) => { + const items = { ...window.localStorage } + const draftKeys = Object.keys(items).filter((key) => + key.startsWith('geonetwork-ui-draft-') + ) + draftKeys.forEach((key) => window.localStorage.removeItem(key)) + cy.log(`Cleared ${draftKeys.length} draft(s).`) + }) + cy.reload() +}) + // -- This is a parent command -- // Cypress.Commands.add('login', (email, password) => { ... }) // From 0af639709ecf55520db6925fb204b5ec95574d46 Mon Sep 17 00:00:00 2001 From: Olivia Guyot Date: Mon, 22 Jul 2024 17:27:55 +0200 Subject: [PATCH 3/7] feat(support): handle GN versions 4.4+, use ES 7.17 minor improvements to init service to improve readability --- support-services/.env | 1 + support-services/docker-compose.yml | 25 +++++++++++++------ .../docker-entrypoint.d/04-upload-thesauri.sh | 1 + 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/support-services/.env b/support-services/.env index 4ee8267eca..9438647d17 100644 --- a/support-services/.env +++ b/support-services/.env @@ -1 +1,2 @@ GEONETWORK_VERSION=4.2.2 +ELASTICSEARCH_VERSION=7.17.15 diff --git a/support-services/docker-compose.yml b/support-services/docker-compose.yml index 5bcc64c8ea..783cf1563b 100644 --- a/support-services/docker-compose.yml +++ b/support-services/docker-compose.yml @@ -21,7 +21,7 @@ services: - ./docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:7.15.1 + image: docker.elastic.co/elasticsearch/elasticsearch:${ELASTICSEARCH_VERSION} ulimits: memlock: soft: -1 @@ -80,15 +80,26 @@ services: GEONETWORK_DB_NAME: geonetwork GEONETWORK_DB_USERNAME: geonetwork GEONETWORK_DB_PASSWORD: geonetwork + DATA_DIR: /catalogue-data + VIRTUAL_HOST: localhost + JAVA_OPTS: > -Dorg.eclipse.jetty.annotations.AnnotationParser.LEVEL=OFF -Djava.security.egd=file:/dev/./urandom -Djava.awt.headless=true -Xms512M -Xss512M -Xmx2G -XX:+UseConcMarkSweepGC - -Dgeonetwork.resources.dir=/var/lib/jetty/webapps/geonetwork/WEB-INF/data/data/resources - -Dgeonetwork.data.dir=/var/lib/jetty/webapps/geonetwork/WEB-INF/data/data - -Dgeonetwork.codeList.dir=/var/lib/jetty/webapps/geonetwork/WEB-INF/data/config/codelist - -Dgeonetwork.schema.dir=/var/lib/jetty/webapps/geonetwork/WEB-INF/data/config/schema_plugins -Xdebug -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=0.0.0.0:5005 + -Dgeonetwork.resources.dir=/catalogue-data/data/resources + -Dgeonetwork.data.dir=/catalogue-data/data + -Dgeonetwork.codeList.dir=/catalogue-data/config/codelist + -Dgeonetwork.schema.dir=/catalogue-data/config/schema_plugins + -Dgeonetwork.config.dir=/catalogue-data/config + -Dgeonetwork.indexConfig.dir=/catalogue-data/config/index + -Des.host=elasticsearch + -Des.protocol=http + -Des.port=9200 + -Des.url=http://elasticsearch:9200 + -Des.username= + -Des.password= depends_on: database: condition: service_healthy @@ -104,7 +115,7 @@ services: timeout: 10s retries: 10 volumes: - - geonetwork_data:/var/lib/jetty/webapps/geonetwork/WEB-INF/data/ + - geonetwork_data:/catalogue-data/ ports: - '8080:8080' - '5005:5005' @@ -123,7 +134,7 @@ services: init: image: alpine/curl # only run init if volumes were cleared - command: sh -c "if [ ! -f /done ]; then run-parts /docker-entrypoint.d --exit-on-error; else echo 'Nothing to do.'; exit 0; fi" + command: sh -c -e "if [ ! -f /done ]; then run-parts /docker-entrypoint.d --exit-on-error; else echo 'Nothing to do.'; exit 0; fi" environment: GEONETWORK_VERSION: ${GEONETWORK_VERSION} depends_on: diff --git a/support-services/docker-entrypoint.d/04-upload-thesauri.sh b/support-services/docker-entrypoint.d/04-upload-thesauri.sh index 738b0eed69..ae898293be 100755 --- a/support-services/docker-entrypoint.d/04-upload-thesauri.sh +++ b/support-services/docker-entrypoint.d/04-upload-thesauri.sh @@ -15,5 +15,6 @@ do -H 'Accept: application/json, text/plain, */*' \ -H "Cookie: JSESSIONID=$jsessionid; XSRF-TOKEN=$xsrf_token" \ -H "X-XSRF-TOKEN: $xsrf_token" + echo "" done From 7664e9f5d8d8b99dcaa107ccab68b551e833c2a7 Mon Sep 17 00:00:00 2001 From: Olivia Guyot Date: Tue, 23 Jul 2024 11:24:19 +0200 Subject: [PATCH 4/7] ci: run e2e tests with various GN versions + misc improvements to the workflow --- .github/workflows/artifacts.yml | 6 ++-- .github/workflows/checks.yml | 52 ++++++++++++++++------------- .github/workflows/snyk-security.yml | 2 +- .github/workflows/webcomponents.yml | 2 +- 4 files changed, 33 insertions(+), 29 deletions(-) diff --git a/.github/workflows/artifacts.yml b/.github/workflows/artifacts.yml index ce7c56b06b..73a86dab8a 100644 --- a/.github/workflows/artifacts.yml +++ b/.github/workflows/artifacts.yml @@ -44,7 +44,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ needs.checks.outputs.ref }} # use the PR head ref if applicable; otherwise keep default behaviour persist-credentials: false @@ -57,9 +57,7 @@ jobs: cache: 'npm' - name: Derive appropriate SHAs for base and head for `nx affected` commands - uses: nrwl/nx-set-shas@v2 - with: - main-branch-name: 'main' + uses: nrwl/nx-set-shas@v3 - name: Install dependencies run: npm ci diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 96aeaa13e3..6fa3fe4f24 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -28,7 +28,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: persist-credentials: false fetch-depth: 0 @@ -38,9 +38,7 @@ jobs: node-version: ${{ env.NODE_VERSION }} cache: 'npm' - name: Derive appropriate SHAs for base and head for `nx affected` commands - uses: nrwl/nx-set-shas@v2 - with: - main-branch-name: 'main' + uses: nrwl/nx-set-shas@v3 - run: npm ci - run: npx nx format:check - run: npx nx affected -t lint --parallel=3 @@ -79,7 +77,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: persist-credentials: false fetch-depth: 0 @@ -89,9 +87,7 @@ jobs: node-version: ${{ env.NODE_VERSION }} cache: 'npm' - name: Derive appropriate SHAs for base and head for `nx affected` commands - uses: nrwl/nx-set-shas@v2 - with: - main-branch-name: 'main' + uses: nrwl/nx-set-shas@v3 - run: npm ci - run: npx nx affected -t build --parallel=3 @@ -102,7 +98,7 @@ jobs: steps: - name: Checkout branch - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: persist-credentials: false fetch-depth: 0 @@ -142,13 +138,18 @@ jobs: comment_tag: build-options GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - cypress-run: - name: End-to-end tests + e2e-run: + name: End-to-end tests, GeoNetwork v${{ matrix.gn_version }} runs-on: ubuntu-latest - outputs: - screenshotsUrl: ${{ steps.upload-screenshots.outputs.artifact-url }} + strategy: + fail-fast: false + matrix: + gn_version: [4.2.2, 4.2.8, 4.4.0] steps: - uses: actions/checkout@v4 + with: + persist-credentials: false + fetch-depth: 0 - name: Use Node.js ${{ env.NODE_VERSION }} uses: actions/setup-node@v3 @@ -156,24 +157,29 @@ jobs: node-version: ${{ env.NODE_VERSION }} cache: 'npm' + - name: Derive appropriate SHAs for base and head for `nx affected` commands + uses: nrwl/nx-set-shas@v3 + - name: Create pipeline docker image - run: cd tools && docker build . -f pipelines/Dockerfile -t geonetwork/geonetwork-ui-tools-pipelines:latest + working-directory: tools + run: docker build . -f pipelines/Dockerfile -t geonetwork/geonetwork-ui-tools-pipelines:latest - - name: Build the backend - run: sudo docker-compose -f support-services/docker-compose.yml up -d init + - name: Start up backend support services + env: + GEONETWORK_VERSION: ${{ matrix.gn_version }} + working-directory: support-services + run: docker compose up -d init - - name: Install dependencies - run: | - npm ci + - run: npm ci - - name: Run tests - run: npx nx run-many --target=e2e + - name: Run e2e tests + run: npx nx affected --target=e2e - uses: actions/upload-artifact@v4 if: always() id: upload-screenshots with: - name: cypress-screenshots + name: cypress-screenshots-gn-${{ matrix.gn_version }} path: | apps/datahub-e2e/cypress/screenshots/**/* apps/metadata-editor-e2e/cypress/screenshots/**/* @@ -193,7 +199,7 @@ jobs: steps: - name: Checkout branch - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: persist-credentials: false fetch-depth: 0 diff --git a/.github/workflows/snyk-security.yml b/.github/workflows/snyk-security.yml index e114be3fed..57461efd08 100644 --- a/.github/workflows/snyk-security.yml +++ b/.github/workflows/snyk-security.yml @@ -35,7 +35,7 @@ jobs: security-events: write # for github/codeql-action/upload-sarif to upload SARIF results runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Run Snyk to check for vulnerabilities uses: snyk/actions/node@master diff --git a/.github/workflows/webcomponents.yml b/.github/workflows/webcomponents.yml index e090eb78bb..fbcea28878 100644 --- a/.github/workflows/webcomponents.yml +++ b/.github/workflows/webcomponents.yml @@ -26,7 +26,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ needs.checks.outputs.ref }} persist-credentials: false From 9696425a9fc932d33b996ca0291861e1fb79de53 Mon Sep 17 00:00:00 2001 From: Olivia Guyot Date: Tue, 23 Jul 2024 11:38:21 +0200 Subject: [PATCH 5/7] feat(es): improve md-quality pipeline to handle multilingual orgs --- tools/pipelines/register-es-pipelines.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tools/pipelines/register-es-pipelines.js b/tools/pipelines/register-es-pipelines.js index 5acb18ccfd..d61d3ea77b 100644 --- a/tools/pipelines/register-es-pipelines.js +++ b/tools/pipelines/register-es-pipelines.js @@ -69,9 +69,14 @@ if(ctx.resourceTitleObject != null && ctx.resourceTitleObject.default != null && if(ctx.resourceAbstractObject != null && ctx.resourceAbstractObject.default != null && ctx.resourceAbstractObject.default != '') { ok++ } +// this checks for single-language Organizations (GN 4.2.2) if(ctx.contact != null && ctx.contact.length > 0 && ctx.contact[0].organisation != null && ctx.contact[0].organisation != '') { ok++ } +// this checks for multilingual Organizations (GN 4.2.3+) +if(ctx.contact != null && ctx.contact.length > 0 && ctx.contact[0].organisationObject != null && ctx.contact[0].organisationObject.default != '') { + ok++ +} if(ctx.contact != null && ctx.contact.length > 0 && ctx.contact[0].email != null && ctx.contact[0].email != '') { ok++ } From e848b2cfec0c21babd72002228cd478853673a40 Mon Sep 17 00:00:00 2001 From: Olivia Guyot Date: Tue, 23 Jul 2024 12:18:19 +0200 Subject: [PATCH 6/7] chore(e2e): execute tests on different ports for each app --- apps/datahub-e2e/project.json | 3 ++- apps/metadata-editor-e2e/project.json | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/apps/datahub-e2e/project.json b/apps/datahub-e2e/project.json index f188bfa518..7a3d5997a3 100644 --- a/apps/datahub-e2e/project.json +++ b/apps/datahub-e2e/project.json @@ -10,7 +10,8 @@ "cypressConfig": "apps/datahub-e2e/cypress.config.js", "devServerTarget": "datahub:serve:development", "testingType": "e2e", - "browser": "chrome" + "browser": "chrome", + "port": "cypress-auto" }, "configurations": { "production": { diff --git a/apps/metadata-editor-e2e/project.json b/apps/metadata-editor-e2e/project.json index fe64bd1e65..306cdf0d07 100644 --- a/apps/metadata-editor-e2e/project.json +++ b/apps/metadata-editor-e2e/project.json @@ -10,7 +10,8 @@ "cypressConfig": "apps/metadata-editor-e2e/cypress.config.js", "devServerTarget": "metadata-editor:serve:development", "testingType": "e2e", - "browser": "chrome" + "browser": "chrome", + "port": "cypress-auto" }, "configurations": { "production": { From 583c63fd84a4878a8419e3a677cfa37ce7ee9366 Mon Sep 17 00:00:00 2001 From: Olivia Guyot Date: Wed, 24 Jul 2024 10:26:05 +0200 Subject: [PATCH 7/7] wip --- .github/workflows/checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 6fa3fe4f24..c6cc169081 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -168,7 +168,7 @@ jobs: env: GEONETWORK_VERSION: ${{ matrix.gn_version }} working-directory: support-services - run: docker compose up -d init + run: docker compose up --quiet-pull init - run: npm ci