dev: validate test configurations (#4520)

go.mod

@@ -85,6 +85,7 @@ require (
 	github.com/ryancurrah/gomodguard v1.3.0
 	github.com/ryanrolds/sqlclosecheck v0.5.1
 	github.com/sanposhiho/wastedassign/v2 v2.0.7
+	github.com/santhosh-tekuri/jsonschema/v5 v5.3.1
 	github.com/sashamelentyev/interfacebloat v1.1.0
 	github.com/sashamelentyev/usestdlibvars v1.25.0
 	github.com/securego/gosec/v2 v2.19.0

go.sum

@@ -466,6 +466,8 @@ github.com/ryanrolds/sqlclosecheck v0.5.1 h1:dibWW826u0P8jNLsLN+En7+RqWWTYrjCB9f
 github.com/ryanrolds/sqlclosecheck v0.5.1/go.mod h1:2g3dUjoS6AL4huFdv6wn55WpLIDjY7ZgUR4J8HOO/XQ=
 github.com/sanposhiho/wastedassign/v2 v2.0.7 h1:J+6nrY4VW+gC9xFzUc+XjPD3g3wF3je/NsJFwFK7Uxc=
 github.com/sanposhiho/wastedassign/v2 v2.0.7/go.mod h1:KyZ0MWTwxxBmfwn33zh3k1dmsbF2ud9pAAGfoLfjhtI=
+github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 h1:lZUw3E0/J3roVtGQ+SCrUrg3ON6NgVqpn3+iol9aGu4=
+github.com/santhosh-tekuri/jsonschema/v5 v5.3.1/go.mod h1:uToXkOrWAZ6/Oc07xWQrPOhJotwFIyu2bBVN41fcDUY=
 github.com/sashamelentyev/interfacebloat v1.1.0 h1:xdRdJp0irL086OyW1H/RTZTr1h/tMEOsumirXcOJqAw=
 github.com/sashamelentyev/interfacebloat v1.1.0/go.mod h1:+Y9yU5YdTkrNvoX0xHc84dxiN1iBi9+G8zZIhPVoNjQ=
 github.com/sashamelentyev/usestdlibvars v1.25.0 h1:IK8SI2QyFzy/2OD2PYnhy84dpfNo9qADrRt6LH8vSzU=

jsonschema/golangci.next.jsonschema.json

@@ -385,15 +385,18 @@
       "additionalProperties": false
     }
   },
+  "type": "object",
+  "additionalProperties": false,
   "properties": {
     "run": {
       "description": "Options for analysis running,",
       "type": "object",
+      "additionalProperties": false,
       "properties": {
         "concurrency": {
           "description": "Number of concurrent runners. Defaults to the number of available CPU cores.",
           "type": "integer",
-          "minimum": 1,
+          "minimum": 0,
           "examples": [4]
         },
         "timeout": {
@@ -441,12 +444,12 @@
           "type": "string",
           "default": "1.17"
         }
-      },
+      }
-      "additionalProperties": false
     },
     "output": {
       "description": "Output configuration options.",
       "type": "object",
+      "additionalProperties": false,
       "properties": {
         "format": {
           "description": "Output format to use.",
@@ -494,12 +497,12 @@
           "type": "boolean",
           "default": true
         }
-      },
+      }
-      "additionalProperties": false
     },
     "linters-settings": {
       "description": "All available settings of specific linters.",
       "type": "object",
+      "additionalProperties": false,
       "properties": {
         "dupword": {
           "type": "object",
@@ -683,8 +686,8 @@
               "patternProperties": {
                 "^[^.]+$": {
                   "description": "Name of a rule.",
-                  "additionalProperties": false,
                   "type": "object",
+                  "additionalProperties": false,
                   "properties": {
                     "list-mode": {
                       "description": "Used to determine the package matching priority.",
@@ -709,11 +712,10 @@
                     },
                     "deny": {
                       "description": "Packages that are not allowed where the value is a suggestion.",
-                      "additionalProperties": false,
                       "type": "array",
                       "items": {
-                        "additionalProperties": false,
                         "type": "object",
+                        "additionalProperties": false,
                         "properties": {
                           "desc": {
                             "description": "Description",
@@ -936,6 +938,7 @@
                   },
                   {
                     "type": "object",
+                    "additionalProperties": false,
                     "properties": {
                       "p": {
                         "description": "Pattern",
@@ -949,8 +952,7 @@
                         "description": "Message",
                         "type": "string"
                       }
-                    },
+                    }
-                    "additionalProperties": false
                   }
                 ]
               }
@@ -1236,6 +1238,7 @@
               "type": "array",
               "items": {
                 "type": "object",
+                "additionalProperties": false,
                 "properties": {
                   "pattern": {
                     "type": "string"
@@ -1243,8 +1246,7 @@
                   "replacement": {
                     "type": "string"
                   }
-                },
+                }
-                "additionalProperties": false
               }
             }
           }
@@ -1282,11 +1284,10 @@
         "goheader": {
           "type": "object",
           "additionalProperties": false,
-          "allOf": [
-            {
           "properties": {
             "values": {
               "type": "object",
+              "additionalProperties": false,
               "properties": {
                 "const": {
                   "description": "Constants to use in the template.",
@@ -1308,12 +1309,12 @@
                 "regexp": {
                   "description": "Regular expressions to use in your template.",
                   "type": "object",
+                  "additionalProperties": false,
                   "patternProperties": {
                     "^.+$": {
                       "type": "string"
                     }
                   },
-                      "additionalProperties": false,
                   "examples": [
                     {
                       "AUTHOR": ".*@mycompany\\.com"
@@ -1321,35 +1322,23 @@
                   ]
                 }
               }
-                }
-              }
             },
-            {
-              "oneOf": [
-                {
-                  "properties": {
             "template": {
               "description": "Template to put on top of every file.",
               "type": "string",
               "examples": [
                 "{{ MY COMPANY }}\nSPDX-License-Identifier: Apache-2.0\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at:\n\n    http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License."
               ]
-                    }
             },
-                  "required": ["template"]
-                },
-                {
-                  "properties": {
             "template-path": {
               "description": "Path to the file containing the template source.",
               "type": "string",
               "examples": ["my_header_template.txt"]
             }
           },
-                  "required": ["template-path"]
+          "oneOf": [
-                }
+            { "required": ["template"] },
-              ]
+            { "required": ["template-path"] }
-            }
           ]
         },
         "goimports": {
@@ -1441,6 +1430,7 @@
           "properties": {
             "allowed": {
               "type": "object",
+              "additionalProperties": false,
               "properties": {
                 "modules": {
                   "description": "List of allowed modules.",
@@ -1462,6 +1452,7 @@
             },
             "blocked": {
               "type": "object",
+              "additionalProperties": false,
               "properties": {
                 "modules": {
                   "description": "List of blocked modules.",
@@ -1471,6 +1462,7 @@
                     "patternProperties": {
                       "^.+$": {
                         "type": "object",
+                        "additionalProperties": false,
                         "properties": {
                           "recommendations": {
                             "description": "Recommended modules that should be used instead.",
@@ -1497,6 +1489,7 @@
                     "patternProperties": {
                       "^.*$": {
                         "type": "object",
+                        "additionalProperties": false,
                         "properties": {
                           "version": {
                             "description": "Version constraint.",
@@ -1727,6 +1720,7 @@
               "type": "array",
               "items": {
                 "type": "object",
+                "additionalProperties": false,
                 "properties": {
                   "pkg": {
                     "description": "Package path e.g. knative.dev/serving/pkg/apis/autoscaling/v1alpha1",
@@ -1931,6 +1925,7 @@
               "type": "array",
               "items": {
                 "type": "object",
+                "additionalProperties": false,
                 "properties": {
                   "name": {
                     "type": "string"
@@ -1941,8 +1936,7 @@
                   "arg-pos": {
                     "type": "integer"
                   }
-                },
+                }
-                "additionalProperties": false
               }
             }
           }
@@ -2578,6 +2572,7 @@
           "properties": {
             "case": {
               "type": "object",
+              "additionalProperties": false,
               "properties": {
                 "use-field-name": {
                   "description": "Use the struct field name to check the name of the struct tag.",
@@ -2772,6 +2767,7 @@
             },
             "benchmark": {
               "type": "object",
+              "additionalProperties": false,
               "properties": {
                 "begin": {
                   "description": "Check if `b.Helper()` begins helper function.",
@@ -2792,6 +2788,7 @@
             },
             "tb": {
               "type": "object",
+              "additionalProperties": false,
               "properties": {
                 "begin": {
                   "description": "Check if `tb.Helper()` begins helper function.",
@@ -2812,6 +2809,7 @@
             },
             "fuzz": {
               "type": "object",
+              "additionalProperties": false,
               "properties": {
                 "begin": {
                   "description": "Check if `f.Helper()` begins helper function.",
@@ -3210,15 +3208,24 @@
                   "type": "object"
                 }
               },
-              "required": ["path"]
+              "oneOf": [
-            }
+                {
-          }
+                  "properties": {
+                    "type": {"enum": ["module"] }
                   }
                 },
-      "additionalProperties": false
+                {
+                  "required": ["path"]
+                }
+              ]
+            }
+          }
+        }
+      }
     },
     "linters": {
       "type": "object",
+      "additionalProperties": false,
       "properties": {
         "enable": {
           "description": "List of enabled linters.",
@@ -3270,11 +3277,11 @@
           "type": "boolean",
           "default": false
         }
-      },
+      }
-      "additionalProperties": false
     },
     "issues": {
       "type": "object",
+      "additionalProperties": false,
       "properties": {
         "exclude": {
           "description": "List of regular expressions of issue texts to exclude.\nBut independently from this option we use default exclude patterns. Their usage can be controlled through `exclude-use-default`.",
@@ -3288,6 +3295,7 @@
           "type": "array",
           "items": {
             "type": "object",
+            "additionalProperties": false,
             "properties": {
               "path": {
                 "type": "string"
@@ -3396,11 +3404,11 @@
           "type": "boolean",
           "default": false
         }
-      },
+      }
-      "additionalProperties": false
     },
     "severity": {
       "type": "object",
+      "additionalProperties": false,
       "properties": {
         "default-severity": {
           "description": "Set the default severity for issues. If severity rules are defined and the issues do not match or no severity is provided to the rule this will be the default severity applied. Severities should match the supported severity names of the selected out format.",
@@ -3446,9 +3454,7 @@
           "default": []
         }
       },
-      "required": ["default-severity"],
+      "required": ["default-severity"]
-      "additionalProperties": false
+    }
   }
-  },
-  "type": "object"
 }

test/configuration_file_test.go

@@ -0,0 +1,115 @@
+package test
+
+import (
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/santhosh-tekuri/jsonschema/v5"
+	"github.com/stretchr/testify/require"
+	"gopkg.in/yaml.v3"
+)
+
+func Test_validateTestConfigurationFiles(t *testing.T) {
+	err := validateTestConfigurationFiles("../jsonschema/golangci.next.jsonschema.json", ".")
+	require.NoError(t, err)
+}
+
+func validateTestConfigurationFiles(schemaPath, targetDir string) error {
+	schema, err := loadSchema(filepath.FromSlash(schemaPath))
+	if err != nil {
+		return fmt.Errorf("load schema: %w", err)
+	}
+
+	yamlFiles, err := findConfigurationFiles(filepath.FromSlash(targetDir))
+	if err != nil {
+		return fmt.Errorf("find configuration files: %w", err)
+	}
+
+	var errAll error
+	for _, filename := range yamlFiles {
+		// internal tests
+		if filename == filepath.FromSlash("testdata/withconfig/.golangci.yml") {
+			continue
+		}
+
+		m, err := decodeYamlFile(filename)
+		if err != nil {
+			return err
+		}
+
+		err = schema.Validate(m)
+		if err != nil {
+			abs, _ := filepath.Abs(filename)
+			errAll = errors.Join(errAll, fmt.Errorf("%s: %w", abs, err))
+		}
+	}
+
+	return errAll
+}
+
+func loadSchema(schemaPath string) (*jsonschema.Schema, error) {
+	compiler := jsonschema.NewCompiler()
+	compiler.Draft = jsonschema.Draft7
+
+	schemaFile, err := os.Open(schemaPath)
+	if err != nil {
+		return nil, fmt.Errorf("open schema file: %w", err)
+	}
+
+	defer func() { _ = schemaFile.Close() }()
+
+	err = compiler.AddResource(filepath.Base(schemaPath), schemaFile)
+	if err != nil {
+		return nil, fmt.Errorf("add schema resource: %w", err)
+	}
+
+	schema, err := compiler.Compile(filepath.Base(schemaPath))
+	if err != nil {
+		return nil, fmt.Errorf("compile schema: %w", err)
+	}
+
+	return schema, nil
+}
+
+func findConfigurationFiles(targetDir string) ([]string, error) {
+	var yamlFiles []string
+
+	err := filepath.WalkDir(targetDir, func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if !d.IsDir() && (strings.EqualFold(filepath.Ext(d.Name()), ".yml") || strings.EqualFold(filepath.Ext(d.Name()), ".yaml")) {
+			yamlFiles = append(yamlFiles, path)
+		}
+
+		return nil
+	})
+	if err != nil {
+		return nil, fmt.Errorf("walk dir: %w", err)
+	}
+
+	return yamlFiles, nil
+}
+
+func decodeYamlFile(filename string) (any, error) {
+	yamlFile, err := os.Open(filename)
+	if err != nil {
+		return nil, fmt.Errorf("[%s] file open: %w", filename, err)
+	}
+
+	defer func() { _ = yamlFile.Close() }()
+
+	var m any
+	err = yaml.NewDecoder(yamlFile).Decode(&m)
+	if err != nil {
+		return nil, fmt.Errorf("[%s] yaml decode: %w", filename, err)
+	}
+
+	return m, nil
+}

test/testdata/configs/govet_fieldalignment.yml

@@ -1,3 +1,4 @@
 linters-settings:
   govet:
-    enable: fieldalignment
+    enable:
+    - fieldalignment

test/testdata/configs/govet_ifaceassert.yml

@@ -1,3 +1,4 @@
 linters-settings:
   govet:
-    enable: ifaceassert
+    enable:
+    - ifaceassert

test/testdata/configs/importas_noalias.yml

@@ -1,4 +0,0 @@
-linters-settings:
-  importas:
-    fff: fmt
-    std_os: os

test/testdata/configs/testifylint_bool_compare_only.yml

@@ -1,6 +1,7 @@
 linters-settings:
   testifylint:
     disable-all: true
-    enable: bool-compare
+    enable:
+      - bool-compare
     bool-compare:
       ignore-custom-types: true

test/testdata/configs/testifylint_require_error_only.yml

@@ -1,6 +1,7 @@
 linters-settings:
   testifylint:
     disable-all: true
-    enable: require-error
+    enable:
+      - require-error
     require-error:
       fn-pattern: ^NoError$

test/testdata/importas_noalias.go

@@ -1,16 +0,0 @@
-//golangcitest:args -Eimportas
-//golangcitest:config_path testdata/configs/importas_noalias.yml
-//golangcitest:expected_exitcode 0
-package testdata
-
-import (
-	wrong_alias "fmt"
-	"os"
-	wrong_alias_again "os"
-)
-
-func ImportAsNoAlias() {
-	wrong_alias.Println("foo")
-	wrong_alias_again.Stdout.WriteString("bar")
-	os.Stdout.WriteString("test")
-}

test/testdata_etc/unused_exported/golangci.yml

@@ -2,6 +2,3 @@ linters:
   disable-all: true
   enable:
     - unused
-linters-settings:
-  unused:
-    check-exported: true