[Backport 2.x] Added a new line linter (opensearch-project#2892)

* Added a new line linter (opensearch-project#2875)

* Added linter to add new line

Signed-off-by: Owais Kazi <[email protected]>

* Fixed new lines

Signed-off-by: Owais Kazi <[email protected]>

* Ignore empty files

Signed-off-by: Owais Kazi <[email protected]>

* Updated DEVELOPER GUIDE

Signed-off-by: Owais Kazi <[email protected]>

* Renamed workflow file

Signed-off-by: Owais Kazi <[email protected]>

* Fixed failing tests

Signed-off-by: Owais Kazi <[email protected]>
(cherry picked from commit 3c5d997)

* Fixed new lines

Signed-off-by: Owais Kazi <[email protected]>

Co-authored-by: Owais Kazi <[email protected]>

.ci/dockerOnLinuxExclusions

@@ -12,4 +12,4 @@ ol-7.7
 sles-12.3 # older version used in Vagrant image
 sles-12.5
 sles-15.1
-sles-15.2
+sles-15.2

.github/workflows/code-hygiene.yml

@@ -0,0 +1,14 @@
+name: Code Hygiene
+
+on: [push, pull_request]
+
+jobs:
+  linelint:
+    runs-on: ubuntu-latest
+    name: Check if all files end in newline
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Linelint
+        uses: fernandrone/[email protected]

.github/workflows/delete_backport_branch.yml

@@ -12,4 +12,4 @@ jobs:
       - name: Delete merged branch
         uses: SvanBoxel/delete-merged-branch@main
         env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/links.yml

@@ -16,4 +16,4 @@ jobs:
           args: --accept=200,403,429 --exclude-mail **/*.html **/*.md **/*.txt **/*.json --exclude-file .lychee.excludes
           fail: true
         env:
-          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
+          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}

.github/workflows/wrapper.yml

@@ -7,4 +7,4 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - uses: gradle/wrapper-validation-action@v1
+      - uses: gradle/wrapper-validation-action@v1

.linelint.yml

@@ -0,0 +1,49 @@
+# 'true' will fix files
+autofix: true
+
+ignore:
+  - .git/
+  - .gradle/
+  - .idea/
+  - '*.sha1'
+  - '*.txt'
+  - '.github/CODEOWNERS'
+  - 'buildSrc/src/testKit/opensearch.build/LICENSE'
+  - 'buildSrc/src/testKit/opensearch.build/NOTICE'
+  - 'server/licenses/apache-log4j-extras-DEPENDENCIES'
+  # Empty files
+  - 'doc-tools/missing-doclet/bin/main/org/opensearch/missingdoclet/MissingDoclet.class'
+  - 'buildSrc/src/integTest/resources/org/opensearch/gradle/internal/fake_git/remote/build.gradle'
+  - 'buildSrc/src/integTest/resources/org/opensearch/gradle/internal/fake_git/remote/distribution/archives/oss-darwin-tar/build.gradle'
+  - 'buildSrc/src/integTest/resources/org/opensearch/gradle/internal/fake_git/remote/distribution/bwc/bugfix/build.gradle'
+  - 'buildSrc/src/integTest/resources/org/opensearch/gradle/internal/fake_git/remote/distribution/bwc/minor/build.gradle'
+  - 'buildSrc/src/main/resources/buildSrc.marker'
+  - 'buildSrc/src/testKit/opensearch-build-resources/settings.gradle'
+  - 'buildSrc/src/testKit/opensearch.build/settings.gradle'
+  - 'buildSrc/src/testKit/reaper/settings.gradle'
+  - 'buildSrc/src/testKit/symbolic-link-preserving-tar/settings.gradle'
+  - 'buildSrc/src/testKit/testingConventions/empty_test_task/.gitignore'
+  - 'client/rest-high-level/src/main/resources/META-INF/services/org.opensearch.plugins.spi.NamedXContentProvider'
+  - 'distribution/bwc/bugfix/build.gradle'
+  - 'distribution/bwc/maintenance/build.gradle'
+  - 'distribution/bwc/minor/build.gradle'
+  - 'distribution/bwc/staged/build.gradle'
+  - 'libs/ssl-config/src/test/resources/certs/pem-utils/empty.pem'
+  - 'qa/evil-tests/src/test/resources/org/opensearch/common/logging/does_not_exist/nothing_to_see_here'
+  - 'qa/os/centos-6/build.gradle'
+  - 'qa/os/debian-8/build.gradle'
+  - 'qa/os/oel-6/build.gradle'
+  - 'qa/os/oel-7/build.gradle'
+  - 'qa/os/sles-12/build.gradle'
+  # Test requires no new line for these files
+  - 'server/src/test/resources/org/opensearch/action/bulk/simple-bulk11.json'
+  - 'server/src/test/resources/org/opensearch/action/search/simple-msearch5.json'
+
+rules:
+  # checks if file ends in a newline character
+  end-of-file:
+    # set to true to enable this rule
+    enable: true
+
+    # if true also checks if file ends in a single newline character
+    single-new-line: true

DEVELOPER_GUIDE.md

@@ -48,6 +48,7 @@
     - [Distributed Framework](#distributed-framework)
   - [Submitting Changes](#submitting-changes)
   - [Backports](#backports)
+  - [LineLint](#linelint)
 
 # Developer Guide
 
@@ -472,3 +473,18 @@ See [CONTRIBUTING](CONTRIBUTING.md).
 ## Backports
 
 The Github workflow in [`backport.yml`](.github/workflows/backport.yml) creates backport PRs automatically when the original PR with an appropriate label `backport <backport-branch-name>` is merged to main with the backport workflow run successfully on the PR. For example, if a PR on main needs to be backported to `1.x` branch, add a label `backport 1.x` to the PR and make sure the backport workflow runs on the PR along with other checks. Once this PR is merged to main, the workflow will create a backport PR to the `1.x` branch.
+
+## LineLint
+A linter in [`code-hygiene.yml`](.github/workflows/code-hygiene.yml) that validates simple newline and whitespace rules in all sorts of files. It can:
+- Recursively check a directory tree for files that do not end in a newline
+- Automatically fix these files by adding a newline or trimming extra newlines.
+
+Rules are defined in `.linelint.yml`.
+
+Executing the binary will automatically search the local directory tree for linting errors.
+
+    linelint .
+
+Pass a list of files or directories to limit your search.
+
+    linelint README.md LICENSE

README.md

@@ -45,4 +45,4 @@ Copyright OpenSearch Contributors. See [NOTICE](NOTICE.txt) for details.
 
 OpenSearch is a registered trademark of Amazon Web Services.
 
-OpenSearch includes certain Apache-licensed Elasticsearch code from Elasticsearch B.V. and other source code. Elasticsearch B.V. is not the source of that other source code. ELASTICSEARCH is a registered trademark of Elasticsearch B.V.
+OpenSearch includes certain Apache-licensed Elasticsearch code from Elasticsearch B.V. and other source code. Elasticsearch B.V. is not the source of that other source code. ELASTICSEARCH is a registered trademark of Elasticsearch B.V.

RELEASING.md

@@ -1,3 +1,3 @@
 ## Releasing
 
-This project follows [OpenSearch project branching, labelling, and releasing](https://github.com/opensearch-project/.github/blob/main/RELEASING.md).
+This project follows [OpenSearch project branching, labelling, and releasing](https://github.com/opensearch-project/.github/blob/main/RELEASING.md).

SECURITY.md

@@ -1,3 +1,3 @@
 ## Reporting a Vulnerability
 
-If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/) or directly via email to [email protected]. Please do **not** create a public GitHub issue.
+If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/) or directly via email to [email protected]. Please do **not** create a public GitHub issue.

buildSrc/src/main/resources/eclipse.settings/org.eclipse.core.resources.prefs

@@ -3,4 +3,4 @@ encoding//src/main/java=UTF-8
 encoding//src/main/resources=UTF-8
 encoding//src/test/java=UTF-8
 encoding//src/test/resources=UTF-8
-encoding/<project>=UTF-8
+encoding/<project>=UTF-8

buildSrc/src/main/resources/minimumGradleVersion

@@ -1 +1 @@
-7.4.1
+7.4.1

buildSrc/src/testKit/testingConventions/build.gradle

@@ -88,6 +88,3 @@ project(':valid_setup_with_base') {
     }
   }
 }
-
-
-

buildSrc/src/testKit/testingConventions/settings.gradle

@@ -16,4 +16,4 @@ include 'all_classes_in_tasks'
 include 'not_implementing_base'
 include 'valid_setup_no_base'
 include 'valid_setup_with_base'
-include 'tests_in_main'
+include 'tests_in_main'

buildSrc/src/testKit/thirdPartyAudit/settings.gradle

@@ -9,4 +9,4 @@
  * GitHub history for details.
  */
 
-include 'sample_jars'
+include 'sample_jars'

dev-tools/prepare_release_update_documentation.py

@@ -136,4 +136,3 @@ def callback(line):
     print('WARNING: no documentation references updates for release %s' % (release_version))
 
   print('*** Done.')
-

dev-tools/signoff-check.sh

@@ -28,4 +28,3 @@ done
 
 # Return non-zero error code if any commits were missing signoff
 exit $missingSignoff
-

distribution/docker/src/test/resources/rest-api-spec/test/10_info.yml

@@ -7,4 +7,3 @@
   - is_true:    version
   - is_true:    version.number
   - match: { version.build_type: "docker" }
-

distribution/docker/src/test/resources/rest-api-spec/test/11_nodes.yml

@@ -123,4 +123,3 @@
   - match:
       $body: |
         /^(\S{5,}\n)+$/
-

distribution/src/bin/opensearch-env-from-file

@@ -47,4 +47,3 @@ for VAR_NAME_FILE in OPENSEARCH_PASSWORD_FILE KEYSTORE_PASSWORD_FILE ; do
     unset "$VAR_NAME_FILE"
   fi
 done
-

distribution/src/bin/opensearch-env.bat

@@ -74,4 +74,3 @@ if defined JAVA_OPTS (
 
 rem check the Java version
 %JAVA% -cp "%OPENSEARCH_CLASSPATH%" "org.opensearch.tools.java_version_checker.JavaVersionChecker" || exit /b 1
-

doc-tools/build.gradle

@@ -8,4 +8,3 @@ version '1.0.0-SNAPSHOT'
 repositories {
     mavenCentral()
 }
-

libs/dissect/build.gradle

@@ -40,4 +40,3 @@ dependencies {
 tasks.named('forbiddenApisMain').configure {
   replaceSignatureFiles 'jdk-signatures'
 }
-

libs/dissect/src/test/resources/specification/tests.json

@@ -360,4 +360,4 @@
     "append": ""
   }
 
-]
+]

libs/geo/build.gradle

@@ -42,4 +42,3 @@ tasks.named('forbiddenApisMain').configure {
   // TODO: Need to decide how we want to handle for forbidden signatures with the changes to core
   replaceSignatureFiles 'jdk-signatures'
 }
-

libs/grok/build.gradle

@@ -44,4 +44,4 @@ tasks.named('forbiddenApisMain').configure {
 
 thirdPartyAudit.ignoreMissingClasses(
   'org.jcodings.unicode.UnicodeCodeRange'
-)
+)

libs/grok/src/main/resources/patterns/exim

@@ -10,4 +10,3 @@ EXIM_PROTOCOL (P=%{NOTSPACE:protocol})
 EXIM_MSG_SIZE (S=%{NUMBER:exim_msg_size})
 EXIM_HEADER_ID (id=%{NOTSPACE:exim_header_id})
 EXIM_SUBJECT (T=%{QS:exim_subject})
-

libs/grok/src/main/resources/patterns/junos

@@ -6,4 +6,3 @@ RT_FLOW1 %{RT_FLOW_EVENT:event}: %{GREEDYDATA:close-reason}: %{IP:src-ip}/%{INT:
 RT_FLOW2 %{RT_FLOW_EVENT:event}: session created %{IP:src-ip}/%{INT:src-port}->%{IP:dst-ip}/%{INT:dst-port} %{DATA:service} %{IP:nat-src-ip}/%{INT:nat-src-port}->%{IP:nat-dst-ip}/%{INT:nat-dst-port} %{DATA:src-nat-rule-name} %{DATA:dst-nat-rule-name} %{INT:protocol-id} %{DATA:policy-name} %{DATA:from-zone} %{DATA:to-zone} %{INT:session-id} .*
 
 RT_FLOW3 %{RT_FLOW_EVENT:event}: session denied %{IP:src-ip}/%{INT:src-port}->%{IP:dst-ip}/%{INT:dst-port} %{DATA:service} %{INT:protocol-id}\(\d\) %{DATA:policy-name} %{DATA:from-zone} %{DATA:to-zone} .*
-

libs/grok/src/main/resources/patterns/postgresql

@@ -1,3 +1,2 @@
 # Default postgresql pg_log format pattern
 POSTGRESQL %{DATESTAMP:timestamp} %{TZ} %{DATA:user_id} %{GREEDYDATA:connection_id} %{POSINT:pid}
-

libs/ssl-config/build.gradle

@@ -61,4 +61,3 @@ tasks.test {
         jvmArgs += ["--add-opens", "java.base/java.security.cert=ALL-UNNAMED"]
     }
 }
-

modules/analysis-common/src/test/resources/org/opensearch/analysis/common/cjk_analysis.json

@@ -34,4 +34,4 @@
          }
       }
    }
-}
+}

modules/analysis-common/src/test/resources/org/opensearch/analysis/common/pattern_capture.json

@@ -43,4 +43,4 @@
          }
       }
    }
-}
+}

modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/analysis-common/60_analysis_scripting.yml

@@ -68,4 +68,3 @@
     - match: { tokens.1.token: "f" }
     - match: { tokens.2.token: "g" }
     - match: { tokens.3.token: "h" }
-

modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/cluster.stats/10_analysis_stats.yml

@@ -119,4 +119,3 @@
   - match: { indices.analysis.built_in_analyzers.2.name: spanish }
   - match: { indices.analysis.built_in_analyzers.2.count: 2 }
   - match: { indices.analysis.built_in_analyzers.2.index_count: 2 }
-

modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/indices.analyze/10_synonyms.yml

@@ -76,4 +76,3 @@
   - match:  { tokens.5.token: dude }
   - match:  { tokens.5.position: 4 }
   - match:  { tokens.5.positionLength: null }
-

modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/search.query/60_synonym_graph.yml

@@ -229,4 +229,3 @@ setup:
                 query: bar baz
                 analyzer: lower_graph_syns
   - match: { hits.total: 1 }
-

modules/analysis-common/src/yamlRestTest/resources/rest-api-spec/test/search.query/70_intervals.yml

@@ -56,4 +56,3 @@ setup:
                         use_field: text_en
                   max_gaps: 1
   - match: { hits.total.value: 1 }
-

modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/220_drop_processor.yml

@@ -91,4 +91,3 @@ teardown:
     get:
       index: test
       id: 3
-

modules/ingest-user-agent/build.gradle

@@ -43,4 +43,3 @@ restResources {
 testClusters.all {
   extraConfigFile 'ingest-user-agent/test-regexes.yml', file('src/test/test-regexes.yml')
 }
-

modules/ingest-user-agent/src/test/test-regexes.yml

@@ -1,3 +1,3 @@
 user_agent_parsers:
   - regex: '.*'
-    family_replacement: 'Test'
+    family_replacement: 'Test'

modules/lang-expression/build.gradle

@@ -52,4 +52,3 @@ tasks.named("dependencyLicenses").configure {
   mapping from: /lucene-.*/, to: 'lucene'
   mapping from: /asm-.*/, to: 'asm'
 }
-

modules/lang-painless/src/main/antlr/PainlessLexer.g4

@@ -124,4 +124,4 @@ ID: [_a-zA-Z] [_a-zA-Z0-9]*;
 mode AFTER_DOT;
 
 DOTINTEGER: ( '0' | [1-9] [0-9]* ) -> mode(DEFAULT_MODE);
-DOTID: [_a-zA-Z] [_a-zA-Z0-9]*     -> mode(DEFAULT_MODE);
+DOTID: [_a-zA-Z] [_a-zA-Z0-9]*     -> mode(DEFAULT_MODE);

modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/100_terms_agg.yml

@@ -139,4 +139,3 @@ setup:
   - is_false: aggregations.placeholder.buckets.0.str_terms.buckets.1.key_as_string
   - match: { aggregations.placeholder.buckets.0.str_terms.buckets.1.doc_count: 1 }
   - match: { aggregations.placeholder.buckets.0.the_bucket_script.value: 2.0 }
-

modules/opensearch-dashboards/build.gradle

@@ -41,4 +41,3 @@ dependencies {
 testClusters.all {
   module ':modules:reindex'
 }
-

modules/repository-url/build.gradle

@@ -75,4 +75,3 @@ testClusters.all {
     "http://snapshot.test*,http://${urlFixture.addressAndPort}"
   }, PropertyNormalization.IGNORE_VALUE
 }
-

modules/systemd/build.gradle

@@ -32,4 +32,3 @@ opensearchplugin {
   description 'Integrates OpenSearch with systemd'
   classname 'org.opensearch.systemd.SystemdPlugin'
 }
-

plugins/analysis-icu/src/test/resources/org/opensearch/index/analysis/KeywordTokenizer.rbbi

@@ -18,4 +18,4 @@
 
 # Apply rule status {200}=RBBI.WORD_LETTER, which is mapped
 # to <ALPHANUM> token type by DefaultICUTokenizerConfig.
-.+ {200};
+.+ {200};

plugins/analysis-kuromoji/build.gradle

@@ -46,4 +46,3 @@ restResources {
 tasks.named("dependencyLicenses").configure {
   mapping from: /lucene-.*/, to: 'lucene'
 }
-

plugins/analysis-phonetic/src/yamlRestTest/resources/rest-api-spec/test/analysis_phonetic/10_metaphone.yml

@@ -31,4 +31,3 @@
     - match:  { tokens.1.token: joe  }
     - match:  { tokens.2.token: BLKS  }
     - match:  { tokens.3.token: bloggs  }
-

plugins/analysis-phonetic/src/yamlRestTest/resources/rest-api-spec/test/analysis_phonetic/20_double_metaphone.yml

@@ -28,4 +28,3 @@
 
     - length: { tokens: 1 }
     - match:  { tokens.0.token: SPRKLF }
-

plugins/analysis-phonetic/src/yamlRestTest/resources/rest-api-spec/test/analysis_phonetic/30_beider_morse.yml

@@ -30,4 +30,3 @@
 
     - length: { tokens: 1 }
     - match:  { tokens.0.token: Svarts }
-

plugins/analysis-phonetic/src/yamlRestTest/resources/rest-api-spec/test/analysis_phonetic/50_daitch_mokotoff.yml

@@ -27,4 +27,3 @@
 
     - length: { tokens: 1 }
     - match:  { tokens.0.token: "645740" }
-

plugins/analysis-smartcn/build.gradle

@@ -47,4 +47,3 @@ restResources {
 tasks.named("dependencyLicenses").configure {
   mapping from: /lucene-.*/, to: 'lucene'
 }
-

plugins/discovery-gce/src/test/resources/org/opensearch/discovery/gce/computeMetadata/v1/project/attributes/google-compute-default-zone

@@ -1 +1 @@
-europe-west1-b
+europe-west1-b

plugins/discovery-gce/src/test/resources/org/opensearch/discovery/gce/computeMetadata/v1/project/project-id

@@ -1 +1 @@
-metadataserver
+metadataserver

plugins/examples/build.gradle

@@ -36,4 +36,3 @@ configure(project('painless-whitelist')) {
     }
   }
 }
-

plugins/examples/custom-settings/build.gradle

@@ -42,4 +42,3 @@ testClusters.all {
   // Adds a setting in the OpenSearch keystore before running the integration tests
   keystore 'custom.secured', 'password'
 }
-

plugins/examples/custom-settings/src/main/config/custom.yml

@@ -2,4 +2,4 @@
 custom:
     simple: foo
     list: [0, 1, 1, 2, 3, 5, 8, 13, 21]
-    filtered: secret
+    filtered: secret