schema: Define JSON schema for taxonomy yaml

.github/schemas/compositional_skills.json

@@ -0,0 +1,54 @@
+{
+    "title": "Compositional Skill",
+    "description": "A compositional skill.",
+    "type": "object",
+    "required": [
+        "created_by",
+        "task_description",
+        "seed_examples"
+    ],
+    "additionalProperties": false,
+    "properties": {
+        "created_by": {
+            "description": "The GitHub username of the contributor.",
+            "type": "string",
+            "minLength": 1
+        },
+        "task_description": {
+            "description": "A description of the skill.",
+            "type": "string",
+            "minLength": 1
+        },
+        "seed_examples": {
+            "description": "An array of seed examples for synthetic data generation.",
+            "type": "array",
+            "minItems": 5,
+            "uniqueItems": true,
+            "items": {
+                "type": "object",
+                "required": [
+                    "question",
+                    "answer"
+                ],
+                "additionalProperties": false,
+                "properties": {
+                    "context": {
+                        "description": "Information that the model is expected to take into account during processing. This is different from knowledge, where the model is expected to gain facts and background knowledge from the tuning process.",
+                        "type": "string",
+                        "minLength": 1
+                    },
+                    "question": {
+                        "description": "A question used for synthetic data generation.",
+                        "type": "string",
+                        "minLength": 1
+                    },
+                    "answer": {
+                        "description": "The desired response for the question.",
+                        "type": "string",
+                        "minLength": 1
+                    }
+                }
+            }
+        }
+    }
+}

.github/schemas/knowledge.json

@@ -0,0 +1,55 @@
+{
+    "title": "Knowledge",
+    "description": "A knowledge skill.",
+    "type": "object",
+    "required": [
+        "created_by",
+        "domain",
+        "task_description",
+        "seed_examples"
+    ],
+    "additionalProperties": false,
+    "properties": {
+        "created_by": {
+            "description": "The GitHub username of the contributor.",
+            "type": "string",
+            "minLength": 1
+        },
+        "domain": {
+            "description": "The knowledge domain.",
+            "type": "string",
+            "minLength": 1
+        },
+        "task_description": {
+            "description": "A description of the skill.",
+            "type": "string",
+            "minLength": 1
+        },
+        "seed_examples": {
+            "description": "An array of seed examples for synthetic data generation.",
+            "type": "array",
+            "minItems": 5,
+            "uniqueItems": true,
+            "items": {
+                "type": "object",
+                "required": [
+                    "question",
+                    "answer"
+                ],
+                "additionalProperties": false,
+                "properties": {
+                    "question": {
+                        "description": "A question used for synthetic data generation.",
+                        "type": "string",
+                        "minLength": 1
+                    },
+                    "answer": {
+                        "description": "The desired response for the question.",
+                        "type": "string",
+                        "minLength": 1
+                    }
+                }
+            }
+        }
+    }
+}

.github/schemas/metadata.json

@@ -0,0 +1,49 @@
+{
+    "title": "Taxonomy metadata",
+    "description": "Accompanying metadata for a peer taxonomy file.",
+    "type": "object",
+    "required": [
+        "seed_examples"
+    ],
+    "additionalProperties": false,
+    "properties": {
+        "seed_examples": {
+            "description": "An array of metadata for the seed examples in the peer taxonomy file.",
+            "type": "array",
+            "minItems": 5,
+            "items": {
+                "type": "object",
+                "required": [
+                    "source",
+                    "license"
+                ],
+                "additionalProperties": false,
+                "properties": {
+                    "source": {
+                        "title": "Attribution Source",
+                        "description": "If information in the context, question, or answer of a seed example come from a 3rd party, for example Wikipedia, then the value must specify a URL to the source material. If the contributor self-authored all the information, then the value must be 'self-authored'.",
+                        "type": "string",
+                        "minLength": 1,
+                        "examples": [
+                            "self-authored",
+                            "https://some.domain/path/to/source/material"
+                        ]
+                    },
+                    "license": {
+                        "title": "Attribution License",
+                        "description": "The value must specify the SPDX License Identifier, https://spdx.org/licenses/, of the source information. See CONTRIBUTING.MD for guidance on acceptable licenses for source information. If the information is self-authored, then 'Apache-2.0' must be used.",
+                        "type": "string",
+                        "minLength": 1,
+                        "examples": [
+                            "CC0-1.0",
+                            "CDLA-Permissive-2.0",
+                            "CC-BY-4.0",
+                            "Apache-2.0",
+                            "MIT"
+                        ]
+                    }
+                }
+            }
+        }
+    }
+}

.github/scripts/check-yaml.sh

@@ -18,28 +18,22 @@ if [ $# -lt 1 ]; then
     exit 1
 fi
 
+SCHEMAS="$(dirname ${BASH_SOURCE[0]})/../schemas"
 CHANGED_FILES="$@"
 ERR=0
-error() { echo "ERROR: $file:$@" 1>&2; ERR=1; }
-warn()  { echo "WARN:  $file:$@" 1>&2; }
+error() { printf "ERROR: %s: %s \"%s\"\n" "$1" "$2" "$3" 1>&2; ERR=1; }
+warn() { printf "WARN: %s: %s \"%s\"\n" "$1" "$2" "$3" 1>&2; }
 for file in ${CHANGED_FILES}; do
-    case $file in knowledge*)
-        error "1:1: We do not accept knowledge PRs at this time"
+    case $file in 
+      compositional_skills/*/qna.yaml)
+        eval "$(check-jsonschema --schemafile $SCHEMAS/compositional_skills.json -o JSON $file | jq -r '.errors[] | (.path | ltrimstr("$")) as $path | "\($path)|line" as $yqline | @sh "$(yq \($yqline) \(.filename))" as $yqcmd | @sh "\(.message[-200:])" as $message | "error \"\(.filename):\($yqcmd):1\" \"\($path)\" \($message)"')"
+        ;;
+      knowledge/*)
+        error "$file:1:1" "." "We do not accept knowledge PRs at this time"
+        ;;
+      *)
+        error "$file:1:1" "." "Taxonomy file must be named 'qna.yaml', not '$(basename $file)'"
+        ;;
     esac
-    if [[ "$file" != *"/qna.yaml" ]]; then
-        error "1:1: Skills file has to be named 'qna.yaml', not '$(basename $file)'"
-    fi
-    yq '.created_by       | length > 0'              $file | grep -q false && error "$(yq '.created_by|line'       $file):1: missing/empty 'created_by'"
-    yq '.task_description | length > 0'              $file | grep -q false && warn  "$(yq '.task_description|line' $file):1: missing/empty 'task_description'"
-    yq '.seed_examples'                              $file | grep -q null  && error "$(yq '.seed_examples|line'    $file):1: missing 'seed_examples'"
-    yq '.seed_examples    | length >= 5'             $file | grep -q false && error "$(yq '.seed_examples|line'    $file):1: less than 5 'seed_examples'"
-    yq '.seed_examples[]  | .question  | length > 0' $file | grep -q false && error "$(yq '.seed_examples|line'    $file):1: missing/empty 'question's"
-    yq '.seed_examples[]  | .answer    | length > 0' $file | grep -q false && error "$(yq '.seed_examples|line'    $file):1: missing/empty 'answer's"
-    if $( yq '.seed_examples[] | has("context")'     $file | grep -q true ); then
-        yq '.seed_examples[] | .context| length > 0' $file | grep -q false && error "$(yq '.seed_examples|line'    $file):1: missing/empty 'context's"
-    fi
-    yq '.seed_examples[].attribution | length > 0'           $file | grep -q false && error "$(yq '.seed_examples|line' $file):1: missing/empty 'attribution's"
-    yq '.seed_examples[].attribution[].source  | length > 0' $file | grep -q false && error "$(yq '.seed_examples|line' $file):1: missing/empty 'attribution source's"
-    yq '.seed_examples[].attribution[].license | length > 0' $file | grep -q false && error "$(yq '.seed_examples|line' $file):1: missing/empty 'attribution license's"
 done
 exit $ERR

.github/workflows/lint.yml

@@ -66,6 +66,7 @@ jobs:
       - name: "Check file contents"
         if: steps.changed-files.outputs.any_changed == 'true'
         run: |
+          pip install check-jsonschema
           echo "::add-matcher::.github/workflows/matchers/lint.json"
           echo
           .github/scripts/check-yaml.sh "${{ steps.changed-files.outputs.all_changed_files }}"