From e57e5ae92f0e2d19ae019bd2f42601c0c0185a55 Mon Sep 17 00:00:00 2001 From: Eric Stroczynski Date: Tue, 21 Jul 2020 10:50:57 -0700 Subject: [PATCH 1/7] This commit modifies the scorecard config to be a componentconfig versioned with other scorecard APIs in `pkg/apis/scorecard/v1alpha3`. It also adds the `generate kustomize scorecard` subcommand, which scaffolds componentconfig bases and patches in `config/scorecard`. `generate kustomize manifests` will add the scorecard kustomize path to the manifests kustomization.yaml, and `generate bundle` will write the scorecard config to `bundle/tests/scorecard/config.yaml` cmd/operator-sdk/generate: add `kustomize scorecard` subcommand and modify `kustomize manifests` subcommand pkg/apis/scorecard/v1alpha3: add ScorecardConfiguration types that map to the original Config's types internal/scorecard: replace Config type with v1alpha3.ScorecardConfiguration and related types --- cmd/operator-sdk/generate/bundle/bundle.go | 32 ++- cmd/operator-sdk/generate/kustomize/cmd.go | 3 + .../generate/kustomize/manifests.go | 13 +- .../generate/kustomize/scorecard/cmd.go | 133 ++++++++++ .../generate/kustomize/scorecard/kustomize.go | 246 ++++++++++++++++++ internal/generate/collector/collect.go | 34 ++- internal/scorecard/config.go | 35 +-- .../bundle/tests/scorecard/config.yaml | 37 +-- internal/scorecard/formatting.go | 10 +- internal/scorecard/labels_test.go | 155 ++++++----- internal/scorecard/run_test.go | 16 +- internal/scorecard/scorecard.go | 30 +-- internal/scorecard/testdata/bundle.tar.gz | Bin 2733 -> 2772 bytes .../bundle/tests/scorecard/config.yaml | 95 +++---- internal/scorecard/testpod.go | 4 +- .../scorecard/v1alpha3/configuration_types.go | 52 ++++ .../v1alpha3/{types.go => test_types.go} | 0 .../advanced-topics/scorecard/scorecard.md | 14 +- .../cli/operator-sdk_generate_kustomize.md | 1 + ...erator-sdk_generate_kustomize_scorecard.md | 56 ++++ 20 files changed, 766 insertions(+), 200 deletions(-) create mode 100644 cmd/operator-sdk/generate/kustomize/scorecard/cmd.go create mode 100644 cmd/operator-sdk/generate/kustomize/scorecard/kustomize.go create mode 100644 pkg/apis/scorecard/v1alpha3/configuration_types.go rename pkg/apis/scorecard/v1alpha3/{types.go => test_types.go} (100%) create mode 100644 website/content/en/docs/cli/operator-sdk_generate_kustomize_scorecard.md diff --git a/cmd/operator-sdk/generate/bundle/bundle.go b/cmd/operator-sdk/generate/bundle/bundle.go index db6639439ad..7a0e5bd9759 100644 --- a/cmd/operator-sdk/generate/bundle/bundle.go +++ b/cmd/operator-sdk/generate/bundle/bundle.go @@ -35,6 +35,7 @@ import ( "github.com/operator-framework/operator-sdk/internal/registry" "github.com/operator-framework/operator-sdk/internal/scorecard" "github.com/operator-framework/operator-sdk/internal/util/projutil" + "github.com/operator-framework/operator-sdk/pkg/apis/scorecard/v1alpha3" ) const ( @@ -218,6 +219,11 @@ func (c bundleCmd) runManifests(cfg *config.Config) (err error) { } } + // Write the scorecard config if it was passed. + if err := writeScorecardConfig(c.outputDir, col.ScorecardConfig); err != nil { + return fmt.Errorf("error writing bundle scorecard config: %v", err) + } + if !c.quiet && !c.stdout { fmt.Println("Bundle manifests generated successfully in", c.outputDir) } @@ -225,6 +231,25 @@ func (c bundleCmd) runManifests(cfg *config.Config) (err error) { return nil } +// writeScorecardConfig writes cfg to dir at the hard-coded config path 'config.yaml'. +func writeScorecardConfig(dir string, cfg v1alpha3.ScorecardConfiguration) error { + if cfg.Metadata.Name == "" { + return nil + } + + b, err := yaml.Marshal(cfg) + if err != nil { + return err + } + + cfgDir := filepath.Join(dir, filepath.FromSlash(scorecard.DefaultConfigDir)) + if err := os.MkdirAll(cfgDir, 0755); err != nil { + return err + } + scorecardConfigPath := filepath.Join(cfgDir, scorecard.ConfigFileName) + return ioutil.WriteFile(scorecardConfigPath, b, 0666) +} + // validateMetadata validates c for bundle metadata generation. func (c bundleCmd) validateMetadata(*config.Config) (err error) { // Ensure a default channel is present. @@ -282,6 +307,8 @@ func (c bundleCmd) generateMetadata(cfg *config.Config, manifestsDir, outputDir return nil } +// NB(estroz): these updates need to be atomic because the bundle's Dockerfile and annotations.yaml +// cannot be out-of-sync. func updateMetadata(cfg *config.Config, bundleRoot string) error { bundleLabels := metricsannotations.MakeBundleMetadataLabels(cfg) for key, value := range scorecardannotations.MakeBundleMetadataLabels(scorecard.DefaultConfigDir) { @@ -292,8 +319,6 @@ func updateMetadata(cfg *config.Config, bundleRoot string) error { } // Write labels to bundle Dockerfile. - // NB(estroz): these "rewrites" need to be atomic because the bundle's Dockerfile and annotations.yaml - // cannot be out-of-sync. if err := rewriteDockerfileLabels(bundle.DockerFile, bundleLabels); err != nil { return fmt.Errorf("error writing LABEL's in %s: %v", bundle.DockerFile, err) } @@ -303,7 +328,8 @@ func updateMetadata(cfg *config.Config, bundleRoot string) error { // Add a COPY for the scorecard config to bundle Dockerfile. // TODO: change input config path to be a flag-based value. - err := writeDockerfileCOPYScorecardConfig(bundle.DockerFile, filepath.FromSlash(scorecard.DefaultConfigDir)) + localScorecardConfigPath := filepath.Join(bundleRoot, filepath.FromSlash(scorecard.DefaultConfigDir)) + err := writeDockerfileCOPYScorecardConfig(bundle.DockerFile, localScorecardConfigPath) if err != nil { return fmt.Errorf("error writing scorecard config COPY in %s: %v", bundle.DockerFile, err) } diff --git a/cmd/operator-sdk/generate/kustomize/cmd.go b/cmd/operator-sdk/generate/kustomize/cmd.go index 30ae4f00f57..6aca12f275e 100644 --- a/cmd/operator-sdk/generate/kustomize/cmd.go +++ b/cmd/operator-sdk/generate/kustomize/cmd.go @@ -16,6 +16,8 @@ package kustomize import ( "github.com/spf13/cobra" + + "github.com/operator-framework/operator-sdk/cmd/operator-sdk/generate/kustomize/scorecard" ) // NewCmd returns the 'kustomize' subcommand. @@ -27,6 +29,7 @@ func NewCmd() *cobra.Command { cmd.AddCommand( newManifestsCmd(), + scorecard.NewCmd(), ) return cmd diff --git a/cmd/operator-sdk/generate/kustomize/manifests.go b/cmd/operator-sdk/generate/kustomize/manifests.go index dd60d01b0e2..42991edd815 100644 --- a/cmd/operator-sdk/generate/kustomize/manifests.go +++ b/cmd/operator-sdk/generate/kustomize/manifests.go @@ -16,6 +16,7 @@ package kustomize import ( "fmt" + "os" "path/filepath" log "github.com/sirupsen/logrus" @@ -174,8 +175,18 @@ func (c manifestsCmd) run(cfg *config.Config) error { return fmt.Errorf("error generating kustomize bases: %v", err) } + // NB(estroz): this is a rather hacky way of adding scorecard componentconfigs to the bundle, + // and won't work if the manifests kustomization.yaml already exists. This should be improved + // with scaffolding markers. + kustomization := manifestsKustomization + + // Add a scorecard kustomization if one exists. + info, err := os.Stat(filepath.Join(filepath.Dir(c.outputDir), "scorecard")) + if err == nil && info.IsDir() { + kustomization += "- ../scorecard\n" + } // Write a kustomization.yaml to outputDir if one does not exist. - if err := kustomize.WriteIfNotExist(c.outputDir, manifestsKustomization); err != nil { + if err := kustomize.WriteIfNotExist(c.outputDir, kustomization); err != nil { return fmt.Errorf("error writing kustomization.yaml: %v", err) } diff --git a/cmd/operator-sdk/generate/kustomize/scorecard/cmd.go b/cmd/operator-sdk/generate/kustomize/scorecard/cmd.go new file mode 100644 index 00000000000..d8269034317 --- /dev/null +++ b/cmd/operator-sdk/generate/kustomize/scorecard/cmd.go @@ -0,0 +1,133 @@ +// Copyright 2020 The Operator-SDK Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scorecard + +import ( + "fmt" + "path/filepath" + + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + "github.com/spf13/pflag" + "sigs.k8s.io/kubebuilder/pkg/model/config" + + kbutil "github.com/operator-framework/operator-sdk/internal/util/kubebuilder" +) + +const scorecardLongHelp = ` +Running 'generate kustomize scorecard' will (re)generate scorecard configuration kustomize bases, +default test patches, and a kustomization.yaml in 'config/scorecard'. +` + +const scorecardExamples = ` + $ operator-sdk generate kustomize scorecard + Generating kustomize files in config/scorecard + Kustomize files generated successfully + $ tree ./config/scorecard + ./config/scorecard/ + ├── bases + │   └── config.yaml + ├── kustomization.yaml + └── patches + ├── basic.config.yaml + └── olm.config.yaml +` + +// defaultTestImageTag points to the latest-released image. +// TODO: change the tag to "latest" once config scaffolding is in a release, +// as the new config spec won't work with the current latest image. +const defaultTestImageTag = "quay.io/operator-framework/scorecard-test:master" + +type scorecardCmd struct { + operatorName string + outputDir string + testImageTag string + quiet bool +} + +// NewCmd returns the `scorecard` subcommand. +func NewCmd() *cobra.Command { + c := &scorecardCmd{} + cmd := &cobra.Command{ + Use: "scorecard", + Short: "Generates scorecard configuration files", + Long: scorecardLongHelp, + Example: scorecardExamples, + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) != 0 { + return fmt.Errorf("command %s doesn't accept any arguments", cmd.CommandPath()) + } + + cfg, err := kbutil.ReadConfig() + if err != nil { + return fmt.Errorf("error reading configuration: %v", err) + } + c.setDefaults(cfg) + + // Run command logic. + if err = c.run(); err != nil { + log.Fatalf("Error generating kustomize files: %v", err) + } + + return nil + }, + } + + c.addFlagsTo(cmd.Flags()) + + return cmd +} + +func (c *scorecardCmd) addFlagsTo(fs *pflag.FlagSet) { + fs.StringVar(&c.operatorName, "operator-name", "", "Name of the operator") + fs.StringVar(&c.outputDir, "output-dir", "", "Directory to write kustomize files") + fs.StringVar(&c.testImageTag, "image", defaultTestImageTag, + "Image to use for default tests; this image must contain the `/scorecard-test` binary") + fs.BoolVarP(&c.quiet, "quiet", "q", false, "Run in quiet mode") + // NB(estroz): might be nice to have an --overwrite flag to explicitly turn on overwrite behavior (the current default). +} + +// defaultDir is the default directory in which to generate kustomize bases and the kustomization.yaml. +var defaultDir = filepath.Join("config", "scorecard") + +// setDefaults sets command defaults. +func (c *scorecardCmd) setDefaults(cfg *config.Config) { + if c.operatorName == "" { + c.operatorName = filepath.Base(cfg.Repo) + } + + if c.outputDir == "" { + c.outputDir = defaultDir + } +} + +// run scaffolds kustomize files for kustomizing a scorecard componentconfig. +func (c scorecardCmd) run() error { + + if !c.quiet { + fmt.Println("Generating kustomize files in", c.outputDir) + } + + err := generate(c.operatorName, c.testImageTag, c.outputDir) + if err != nil { + return err + } + + if !c.quiet { + fmt.Println("Kustomize files generated successfully") + } + + return nil +} diff --git a/cmd/operator-sdk/generate/kustomize/scorecard/kustomize.go b/cmd/operator-sdk/generate/kustomize/scorecard/kustomize.go new file mode 100644 index 00000000000..07672c32fb3 --- /dev/null +++ b/cmd/operator-sdk/generate/kustomize/scorecard/kustomize.go @@ -0,0 +1,246 @@ +// Copyright 2020 The Operator-SDK Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scorecard + +import ( + "bytes" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "text/template" + + "github.com/operator-framework/operator-registry/pkg/registry" + "sigs.k8s.io/yaml" + + "github.com/operator-framework/operator-sdk/internal/scaffold/kustomize" + "github.com/operator-framework/operator-sdk/internal/scorecard" + "github.com/operator-framework/operator-sdk/pkg/apis/scorecard/v1alpha3" +) + +// kustomization.yaml file template for the scorecard componentconfig. This should always be written to +// config/scorecard/kustomization.yaml since it only references files in config. +const scorecardKustomizationTemplate = `resources: +{{- range $i, $path := .ResourcePaths }} +- {{ $path }} +{{- end }} +patchesJson6902: +{{- range $i, $patch := .JSONPatches }} +- path: {{ $patch.Path }} + target: + group: {{ $patch.Target.Group }} + version: {{ $patch.Target.Version }} + kind: {{ $patch.Target.Kind }} + name: {{ $patch.Target.Name }} +{{- end }} +` + +// scorecardKustomization holds data required to generate a scorecard's kustomization.yaml. +type scorecardKustomization struct { + ResourcePaths []string + JSONPatches []kustomizationJSON6902Patch +} + +// kustomizationJSON6902Patch holds path and target data to write a patchesJson6902 list in a kustomization.yaml. +type kustomizationJSON6902Patch struct { + Path string + Target registry.DefinitionKey +} + +const ( + // scorecardConfigName is the default scorecard componentconfig's metadata.name, + // which must be set on all kustomize-able bases. This name is only used for + // `kustomize build` pattern match and not for "creation". + scorecardConfigName = "config" +) + +// generate scaffolds kustomize bundle bases and a kustomization.yaml. +// TODO(estroz): refactor this to be testable (in-mem fs) and easier to read. +func generate(operatorName, testImageTag, outputDir string) error { + + kustomization := scorecardKustomization{} + + // Config bases. + basesDir := filepath.Join(outputDir, "bases") + if err := os.MkdirAll(basesDir, 0755); err != nil { + return err + } + + configBase := newScorecardConfigurationBase() + b, err := yaml.Marshal(configBase) + if err != nil { + return fmt.Errorf("error marshaling default config: %v", err) + } + relBasePath := filepath.Join("bases", scorecard.ConfigFileName) + basePath := filepath.Join(basesDir, scorecard.ConfigFileName) + if err := ioutil.WriteFile(basePath, b, 0666); err != nil { + return fmt.Errorf("error writing default scorecard config: %v", err) + } + kustomization.ResourcePaths = append(kustomization.ResourcePaths, relBasePath) + scorecardConfigTarget := registry.DefinitionKey{ + Group: v1alpha3.SchemeGroupVersion.Group, + Version: v1alpha3.SchemeGroupVersion.Version, + Kind: v1alpha3.ScorecardConfigurationKind, + Name: scorecardConfigName, + } + + // Config patches. + patchesDir := filepath.Join(outputDir, "patches") + if err := os.MkdirAll(patchesDir, 0755); err != nil { + return err + } + + // Basic scorecard tests patch. + basicPatch := newBasicScorecardConfigurationPatch(operatorName, testImageTag) + b, err = yaml.Marshal(basicPatch) + if err != nil { + return fmt.Errorf("error marshaling basic patch config: %v", err) + } + basicPatchFileName := fmt.Sprintf("basic.%s", scorecard.ConfigFileName) + if err := ioutil.WriteFile(filepath.Join(patchesDir, basicPatchFileName), b, 0666); err != nil { + return fmt.Errorf("error writing basic scorecard config patch: %v", err) + } + kustomization.JSONPatches = append(kustomization.JSONPatches, kustomizationJSON6902Patch{ + Path: filepath.Join("patches", basicPatchFileName), + Target: scorecardConfigTarget, + }) + + // OLM scorecard tests patch. + olmPatch := newOLMScorecardConfigurationPatch(operatorName, testImageTag) + b, err = yaml.Marshal(olmPatch) + if err != nil { + return fmt.Errorf("error marshaling OLM patch config: %v", err) + } + olmPatchFileName := fmt.Sprintf("olm.%s", scorecard.ConfigFileName) + if err := ioutil.WriteFile(filepath.Join(patchesDir, olmPatchFileName), b, 0666); err != nil { + return fmt.Errorf("error writing default scorecard config: %v", err) + } + kustomization.JSONPatches = append(kustomization.JSONPatches, kustomizationJSON6902Patch{ + Path: filepath.Join("patches", olmPatchFileName), + Target: scorecardConfigTarget, + }) + + // Write a kustomization.yaml to outputDir if one does not exist. + t, err := template.New("scorecard").Parse(scorecardKustomizationTemplate) + if err != nil { + return fmt.Errorf("error parsing default kustomize template: %v", err) + } + buf := bytes.Buffer{} + if err = t.Execute(&buf, kustomization); err != nil { + return fmt.Errorf("error executing on default kustomize template: %v", err) + } + if err := kustomize.Write(outputDir, buf.String()); err != nil { + return fmt.Errorf("error writing default scorecard kustomization.yaml: %v", err) + } + + return nil +} + +// jsonPatches is a list of JSON patch objects. +type jsonPatches []jsonPatchObject + +// jsonPatchObject is a JSON 6902 patch object specific to the scorecard's test configuration. +// See https://tools.ietf.org/html/rfc6902 for details. +type jsonPatchObject struct { + Op string `json:"op"` + Path string `json:"path"` + Value v1alpha3.TestConfiguration `json:"value"` +} + +// newScorecardConfigurationBase returns a scorecard componentconfig object with one parallel stage. +// The returned object is intended to be marshaled and written to disk as a kustomize base. +func newScorecardConfigurationBase() (cfg v1alpha3.ScorecardConfiguration) { + cfg.SetGroupVersionKind(v1alpha3.SchemeGroupVersion.WithKind(v1alpha3.ScorecardConfigurationKind)) + cfg.Metadata.Name = scorecardConfigName + cfg.Spec.Stages = []v1alpha3.StageConfiguration{ + { + Parallel: true, + Tests: []v1alpha3.TestConfiguration{}, + }, + } + return cfg +} + +func makeTestStageJSONPath(stageIdx, testIdx int) string { + return fmt.Sprintf("/spec/stages/%d/tests/%d", stageIdx, testIdx) +} + +// newBasicScorecardConfigurationPatch returns default "basic" test configurations as JSON patch objects +// to be inserted into the componentconfig base as a first stage test element. +// The returned patches are intended to be marshaled and written to disk as in a kustomize patch file. +func newBasicScorecardConfigurationPatch(operatorName, testImageTag string) (ps jsonPatches) { + for i, cfg := range makeDefaultBasicTestConfigs(operatorName, testImageTag) { + ps = append(ps, jsonPatchObject{ + Op: "add", + Path: makeTestStageJSONPath(0, i), + Value: cfg, + }) + } + return ps +} + +// makeDefaultBasicTestConfigs returns all default "basic" test configurations. +func makeDefaultBasicTestConfigs(operatorName, testImageTag string) (cfgs []v1alpha3.TestConfiguration) { + for _, testName := range []string{"basic-check-spec"} { + cfgs = append(cfgs, v1alpha3.TestConfiguration{ + Image: testImageTag, + Entrypoint: []string{"scorecard-test", testName}, + Labels: map[string]string{ + "operator": operatorName, + "suite": "basic", + "test": fmt.Sprintf("%s-test", testName), + }, + }) + } + + return cfgs +} + +// newOLMScorecardConfigurationPatch returns default "olm" test configurations as JSON patch objects +// to be inserted into the componentconfig base as a first stage test element. +// The returned patches are intended to be marshaled and written to disk as in a kustomize patch file. +func newOLMScorecardConfigurationPatch(operatorName, testImageTag string) (ps jsonPatches) { + for i, cfg := range makeDefaultOLMTestConfigs(operatorName, testImageTag) { + ps = append(ps, jsonPatchObject{ + Op: "add", + Path: makeTestStageJSONPath(0, i), + Value: cfg, + }) + } + return ps +} + +// makeDefaultOLMTestConfigs returns all default "olm" test configurations. +func makeDefaultOLMTestConfigs(operatorName, testImageTag string) (cfgs []v1alpha3.TestConfiguration) { + for _, testName := range []string{ + "olm-bundle-validation", + "olm-crds-have-validation", + "olm-crds-have-resources", + "olm-spec-descriptors", + "olm-status-descriptors"} { + + cfgs = append(cfgs, v1alpha3.TestConfiguration{ + Image: testImageTag, + Entrypoint: []string{"scorecard-test", testName}, + Labels: map[string]string{ + "operator": operatorName, + "suite": "olm", + "test": fmt.Sprintf("%s-test", testName), + }, + }) + } + + return cfgs +} diff --git a/internal/generate/collector/collect.go b/internal/generate/collector/collect.go index 0e672b7d24f..61d7c4cb319 100644 --- a/internal/generate/collector/collect.go +++ b/internal/generate/collector/collect.go @@ -16,6 +16,7 @@ package collector import ( "bytes" + "errors" "fmt" "io" "io/ioutil" @@ -32,6 +33,7 @@ import ( "sigs.k8s.io/yaml" "github.com/operator-framework/operator-sdk/internal/util/k8sutil" + scorecardv1alpha3 "github.com/operator-framework/operator-sdk/pkg/apis/scorecard/v1alpha3" ) // Manifests holds a collector of all manifests relevant to CSV updates. @@ -44,7 +46,9 @@ type Manifests struct { ValidatingWebhooks []admissionregv1.ValidatingWebhook MutatingWebhooks []admissionregv1.MutatingWebhook CustomResources []unstructured.Unstructured - Others []unstructured.Unstructured + ScorecardConfig scorecardv1alpha3.ScorecardConfiguration + + Others []unstructured.Unstructured } // UpdateFromDirs adds Roles, ClusterRoles, Deployments, and Custom Resource examples @@ -85,6 +89,8 @@ func (c *Manifests) UpdateFromDirs(deployDir, crdsDir string) error { err = c.addValidatingWebhookConfigurations(manifest) case "MutatingWebhookConfiguration": err = c.addMutatingWebhookConfigurations(manifest) + case scorecardv1alpha3.ScorecardConfigurationKind: + err = c.addScorecardConfig(manifest) default: err = c.addOthers(manifest) } @@ -145,6 +151,8 @@ func (c *Manifests) UpdateFromReader(r io.Reader) error { err = c.addValidatingWebhookConfigurations(manifest) case "MutatingWebhookConfiguration": err = c.addMutatingWebhookConfigurations(manifest) + case scorecardv1alpha3.ScorecardConfigurationKind: + err = c.addScorecardConfig(manifest) default: err = c.addOthers(manifest) } @@ -167,7 +175,7 @@ func (c *Manifests) UpdateFromReader(r io.Reader) error { return nil } -// addRoles assumes add manifest data in rawManifests are Roles and adds them +// addRoles assumes all manifest data in rawManifests are Roles and adds them // to the collector. func (c *Manifests) addRoles(rawManifests ...[]byte) error { for _, rawManifest := range rawManifests { @@ -180,7 +188,7 @@ func (c *Manifests) addRoles(rawManifests ...[]byte) error { return nil } -// addClusterRoles assumes add manifest data in rawManifests are ClusterRoles +// addClusterRoles assumes all manifest data in rawManifests are ClusterRoles // and adds them to the collector. func (c *Manifests) addClusterRoles(rawManifests ...[]byte) error { for _, rawManifest := range rawManifests { @@ -193,7 +201,7 @@ func (c *Manifests) addClusterRoles(rawManifests ...[]byte) error { return nil } -// addDeployments assumes add manifest data in rawManifests are Deployments +// addDeployments assumes all manifest data in rawManifests are Deployments // and adds them to the collector. func (c *Manifests) addDeployments(rawManifests ...[]byte) error { for _, rawManifest := range rawManifests { @@ -206,7 +214,7 @@ func (c *Manifests) addDeployments(rawManifests ...[]byte) error { return nil } -// addCustomResourceDefinitions assumes add manifest data in rawManifests are +// addCustomResourceDefinitions assumes all manifest data in rawManifests are // CustomResourceDefinitions and adds them to the collector. version determines // which CustomResourceDefinition type is used for all manifests in rawManifests. func (c *Manifests) addCustomResourceDefinitions(version string, rawManifests ...[]byte) (err error) { @@ -257,7 +265,21 @@ func (c *Manifests) addMutatingWebhookConfigurations(rawManifests ...[]byte) err return nil } -// addOthers assumes add manifest data in rawManifests are able to be +// addScorecardConfig assumes manifest data in rawManifests is a ScorecardConfigs and adds it to the collector. +// If a config has already been found, addScorecardConfig will return an error. +func (c *Manifests) addScorecardConfig(rawManifest []byte) error { + cfg := scorecardv1alpha3.ScorecardConfiguration{} + if err := yaml.Unmarshal(rawManifest, &cfg); err != nil { + return err + } + if c.ScorecardConfig.Metadata.Name != "" { + return errors.New("duplicate ScorecardConfigurations in collector input") + } + c.ScorecardConfig = cfg + return nil +} + +// addOthers assumes all manifest data in rawManifests are able to be // unmarshalled into an Unstructured object and adds them to the collector. func (c *Manifests) addOthers(rawManifests ...[]byte) error { for _, rawManifest := range rawManifests { diff --git a/internal/scorecard/config.go b/internal/scorecard/config.go index 9854b70abe4..f87bf94939e 100644 --- a/internal/scorecard/config.go +++ b/internal/scorecard/config.go @@ -18,6 +18,8 @@ import ( "io/ioutil" "sigs.k8s.io/yaml" + + "github.com/operator-framework/operator-sdk/pkg/apis/scorecard/v1alpha3" ) const ( @@ -27,43 +29,20 @@ const ( DefaultConfigDir = "tests/scorecard/" ) -type Stage struct { - Parallel bool `yaml:"parallel"` - Tests []Test `yaml:"tests"` -} - -type Test struct { - // Image is the name of the testimage - Image string `json:"image"` - // Entrypoint is list of commands and arguments passed to the test image - Entrypoint []string `json:"entrypoint,omitempty"` - // Labels that further describe the test and enable selection - Labels map[string]string `json:"labels,omitempty"` -} - -// Config represents the set of test configurations which scorecard -// would run based on user input -type Config struct { - Stages []Stage `yaml:"stages"` -} - // LoadConfig will find and return the scorecard config, the config file // is found from a bundle location (TODO bundle image) // scorecard config.yaml is expected to be in the bundle at the following // location: tests/scorecard/config.yaml // the user can override this location using the --config CLI flag -func LoadConfig(configFilePath string) (Config, error) { - c := Config{} +// TODO: version this. +func LoadConfig(configFilePath string) (v1alpha3.ScorecardConfiguration, error) { + c := v1alpha3.ScorecardConfiguration{} - // TODO handle bundle images, not just on-disk yamlFile, err := ioutil.ReadFile(configFilePath) if err != nil { return c, err } - if err := yaml.Unmarshal(yamlFile, &c); err != nil { - return c, err - } - - return c, nil + err = yaml.Unmarshal(yamlFile, &c) + return c, err } diff --git a/internal/scorecard/examples/custom-scorecard-tests/bundle/tests/scorecard/config.yaml b/internal/scorecard/examples/custom-scorecard-tests/bundle/tests/scorecard/config.yaml index 103a2ec1b4f..944f89d6a5a 100644 --- a/internal/scorecard/examples/custom-scorecard-tests/bundle/tests/scorecard/config.yaml +++ b/internal/scorecard/examples/custom-scorecard-tests/bundle/tests/scorecard/config.yaml @@ -1,16 +1,21 @@ -stages: -- tests: - - image: quay.io/username/custom-scorecard-tests:dev - entrypoint: - - custom-scorecard-tests - - customtest1 - labels: - suite: custom - test: customtest1 - - image: quay.io/username/custom-scorecard-tests:dev - entrypoint: - - custom-scorecard-tests - - customtest2 - labels: - suite: custom - test: customtest2 +kind: ScorecardConfiguration +apiversion: scorecard.operatorframework.io/v1alpha3 +metadata: + name: config +spec: + stages: + - tests: + - image: quay.io/username/custom-scorecard-tests:dev + entrypoint: + - custom-scorecard-tests + - customtest1 + labels: + suite: custom + test: customtest1 + - image: quay.io/username/custom-scorecard-tests:dev + entrypoint: + - custom-scorecard-tests + - customtest2 + labels: + suite: custom + test: customtest2 diff --git a/internal/scorecard/formatting.go b/internal/scorecard/formatting.go index a6b7c0622a3..b32f3f45361 100644 --- a/internal/scorecard/formatting.go +++ b/internal/scorecard/formatting.go @@ -42,15 +42,13 @@ func (r PodTestRunner) getTestStatus(ctx context.Context, p *v1.Pod) (output *v1 // run based on user selection func (o Scorecard) List() v1alpha3.TestList { output := v1alpha3.NewTestList() - for _, stage := range o.Config.Stages { + for _, stage := range o.Config.Spec.Stages { tests := o.selectTests(stage) for _, test := range tests { item := v1alpha3.NewTest() - item.Spec = v1alpha3.TestSpec{ - Image: test.Image, - Entrypoint: test.Entrypoint, - Labels: test.Labels, - } + item.Spec.Image = test.Image + item.Spec.Entrypoint = test.Entrypoint + item.Spec.Labels = test.Labels output.Items = append(output.Items, item) } } diff --git a/internal/scorecard/labels_test.go b/internal/scorecard/labels_test.go index bbdf7050720..6982cf9bbcb 100644 --- a/internal/scorecard/labels_test.go +++ b/internal/scorecard/labels_test.go @@ -17,8 +17,9 @@ package scorecard import ( "testing" - "gopkg.in/yaml.v2" "k8s.io/apimachinery/pkg/labels" + + "github.com/operator-framework/operator-sdk/pkg/apis/scorecard/v1alpha3" ) func TestEmptySelector(t *testing.T) { @@ -26,25 +27,22 @@ func TestEmptySelector(t *testing.T) { cases := []struct { selectorValue string testsSelected int + config v1alpha3.ScorecardConfiguration wantError bool }{ - {"", 7, false}, - {"suite in (kuttl)", 1, false}, - {"test=basic-check-spec-test", 1, false}, - {"testXwriteintocr", 0, false}, - {"test X writeintocr", 0, true}, + {"", 7, testConfig, false}, + {"suite in (kuttl)", 1, testConfig, false}, + {"test=basic-check-spec-test", 1, testConfig, false}, + {"testXwriteintocr", 0, testConfig, false}, + {"test X writeintocr", 0, testConfig, true}, } for _, c := range cases { t.Run(c.selectorValue, func(t *testing.T) { o := Scorecard{} + o.Config = c.config - err := yaml.Unmarshal([]byte(testConfig), &o.Config) - if err != nil { - t.Log(err) - return - } - + var err error o.Selector, err = labels.Parse(c.selectorValue) if err == nil && c.wantError { t.Fatalf("Wanted error but got no error") @@ -55,7 +53,7 @@ func TestEmptySelector(t *testing.T) { return } - tests := o.selectTests(o.Config.Stages[0]) + tests := o.selectTests(o.Config.Spec.Stages[0]) testsSelected := len(tests) if testsSelected != c.testsSelected { t.Errorf("Wanted testsSelected %d, got: %d", c.testsSelected, testsSelected) @@ -65,52 +63,85 @@ func TestEmptySelector(t *testing.T) { } } -const testConfig = `stages: -- tests: - - image: quay.io/someuser/customtest1:v0.0.1 - entrypoint: - - custom-test - labels: - suite: custom - test: customtest1 - - image: quay.io/someuser/customtest2:v0.0.1 - entrypoint: - - custom-test - labels: - suite: custom - test: customtest2 - - image: quay.io/redhat/basictests:v0.0.1 - entrypoint: - - scorecard-test - - basic-check-spec - labels: - suite: basic - test: basic-check-spec-test - - image: quay.io/redhat/basictests:v0.0.1 - entrypoint: - - scorecard-test - - basic-check-status - labels: - suite: basic - test: basic-check-status-test - - image: quay.io/redhat/olmtests:v0.0.1 - entrypoint: - - scorecard-test - - olm-bundle-validation - labels: - suite: olm - test: olm-bundle-validation-test - - image: quay.io/redhat/olmtests:v0.0.1 - entrypoint: - - scorecard-test - - olm-crds-have-validation - labels: - suite: olm - test: olm-crds-have-validation-test - - image: quay.io/redhat/kuttltests:v0.0.1 - labels: - suite: kuttl - entrypoint: - - kuttl-test - - olm-status-descriptors -` +var testConfig = v1alpha3.ScorecardConfiguration{ + Spec: v1alpha3.ScorecardConfigurationSpec{ + Stages: []v1alpha3.StageConfiguration{ + { + Tests: []v1alpha3.TestConfiguration{ + {Image: "quay.io/someuser/customtest1:v0.0.1", + Entrypoint: []string{ + "custom-test", + }, + Labels: map[string]string{ + "suite": "custom", + "test": "customtest1", + }, + }, + + {Image: "quay.io/someuser/customtest2:v0.0.1", + Entrypoint: []string{ + "custom-test", + }, + Labels: map[string]string{ + "suite": "custom", + "test": "customtest2", + }, + }, + + {Image: "quay.io/redhat/basictests:v0.0.1", + Entrypoint: []string{ + "scorecard-test", + "basic-check-spec", + }, + Labels: map[string]string{ + "suite": "basic", + "test": "basic-check-spec-test", + }, + }, + + {Image: "quay.io/redhat/basictests:v0.0.1", + Entrypoint: []string{ + "scorecard-test", + "basic-check-status", + }, + Labels: map[string]string{ + "suite": "basic", + "test": "basic-check-status-test", + }, + }, + + {Image: "quay.io/redhat/olmtests:v0.0.1", + Entrypoint: []string{ + "scorecard-test", + "olm-bundle-validation", + }, + Labels: map[string]string{ + "suite": "olm", + "test": "olm-bundle-validation-test", + }, + }, + + {Image: "quay.io/redhat/olmtests:v0.0.1", + Entrypoint: []string{ + "scorecard-test", + "olm-crds-have-validation", + }, + Labels: map[string]string{ + "suite": "olm", + "test": "olm-crds-have-validation-test", + }, + }, + {Image: "quay.io/redhat/kuttltests:v0.0.1", + Entrypoint: []string{ + "kuttl-test", + "olm-status-descriptors", + }, + Labels: map[string]string{ + "suite": "kuttl", + }, + }, + }, + }, + }, + }, +} diff --git a/internal/scorecard/run_test.go b/internal/scorecard/run_test.go index 5a92aa16ce4..708579f5a9d 100644 --- a/internal/scorecard/run_test.go +++ b/internal/scorecard/run_test.go @@ -153,13 +153,15 @@ func TestRunSequentialFail(t *testing.T) { func getFakeScorecard(parallel bool) Scorecard { return Scorecard{ - Config: Config{ - Stages: []Stage{ - { - Parallel: parallel, - Tests: []Test{ - {}, - {}, + Config: v1alpha3.ScorecardConfiguration{ + Spec: v1alpha3.ScorecardConfigurationSpec{ + Stages: []v1alpha3.StageConfiguration{ + { + Parallel: parallel, + Tests: []v1alpha3.TestConfiguration{ + {}, + {}, + }, }, }, }, diff --git a/internal/scorecard/scorecard.go b/internal/scorecard/scorecard.go index 0011a7f5c7c..2e0734663bd 100644 --- a/internal/scorecard/scorecard.go +++ b/internal/scorecard/scorecard.go @@ -32,12 +32,12 @@ import ( type TestRunner interface { Initialize(context.Context) error - RunTest(context.Context, Test) (*v1alpha3.TestStatus, error) + RunTest(context.Context, v1alpha3.TestConfiguration) (*v1alpha3.TestStatus, error) Cleanup(context.Context) error } type Scorecard struct { - Config Config + Config v1alpha3.ScorecardConfiguration Selector labels.Selector TestRunner TestRunner SkipCleanup bool @@ -67,7 +67,7 @@ func (o Scorecard) Run(ctx context.Context) (v1alpha3.TestList, error) { return testOutput, err } - for _, stage := range o.Config.Stages { + for _, stage := range o.Config.Spec.Stages { tests := o.selectTests(stage) if len(tests) == 0 { continue @@ -93,11 +93,11 @@ func (o Scorecard) Run(ctx context.Context) (v1alpha3.TestList, error) { return testOutput, nil } -func (o Scorecard) runStageParallel(ctx context.Context, tests []Test, results chan<- v1alpha3.Test) { +func (o Scorecard) runStageParallel(ctx context.Context, tests []v1alpha3.TestConfiguration, results chan<- v1alpha3.Test) { var wg sync.WaitGroup for _, t := range tests { wg.Add(1) - go func(test Test) { + go func(test v1alpha3.TestConfiguration) { results <- o.runTest(ctx, test) wg.Done() }(t) @@ -105,32 +105,30 @@ func (o Scorecard) runStageParallel(ctx context.Context, tests []Test, results c wg.Wait() } -func (o Scorecard) runStageSequential(ctx context.Context, tests []Test, results chan<- v1alpha3.Test) { +func (o Scorecard) runStageSequential(ctx context.Context, tests []v1alpha3.TestConfiguration, results chan<- v1alpha3.Test) { for _, test := range tests { results <- o.runTest(ctx, test) } } -func (o Scorecard) runTest(ctx context.Context, test Test) v1alpha3.Test { +func (o Scorecard) runTest(ctx context.Context, test v1alpha3.TestConfiguration) v1alpha3.Test { result, err := o.TestRunner.RunTest(ctx, test) if err != nil { result = convertErrorToStatus(err, "") } out := v1alpha3.NewTest() - out.Spec = v1alpha3.TestSpec{ - Image: test.Image, - Entrypoint: test.Entrypoint, - Labels: test.Labels, - } + out.Spec.Image = test.Image + out.Spec.Entrypoint = test.Entrypoint + out.Spec.Labels = test.Labels out.Status = *result return out } // selectTests applies an optionally passed selector expression // against the configured set of tests, returning the selected tests -func (o *Scorecard) selectTests(stage Stage) []Test { - selected := make([]Test, 0) +func (o *Scorecard) selectTests(stage v1alpha3.StageConfiguration) []v1alpha3.TestConfiguration { + selected := make([]v1alpha3.TestConfiguration, 0) for _, test := range stage.Tests { if o.Selector == nil || o.Selector.String() == "" || o.Selector.Matches(labels.Set(test.Labels)) { // TODO olm manifests check @@ -187,7 +185,7 @@ func (r PodTestRunner) Cleanup(ctx context.Context) (err error) { } // RunTest executes a single test -func (r PodTestRunner) RunTest(ctx context.Context, test Test) (*v1alpha3.TestStatus, error) { +func (r PodTestRunner) RunTest(ctx context.Context, test v1alpha3.TestConfiguration) (*v1alpha3.TestStatus, error) { // Create a Pod to run the test podDef := getPodDefinition(r.configMapName, test, r) pod, err := r.Client.CoreV1().Pods(r.Namespace).Create(ctx, podDef, metav1.CreateOptions{}) @@ -204,7 +202,7 @@ func (r PodTestRunner) RunTest(ctx context.Context, test Test) (*v1alpha3.TestSt } // RunTest executes a single test -func (r FakeTestRunner) RunTest(ctx context.Context, test Test) (result *v1alpha3.TestStatus, err error) { +func (r FakeTestRunner) RunTest(ctx context.Context, test v1alpha3.TestConfiguration) (result *v1alpha3.TestStatus, err error) { select { case <-time.After(r.Sleep): return r.TestStatus, r.Error diff --git a/internal/scorecard/testdata/bundle.tar.gz b/internal/scorecard/testdata/bundle.tar.gz index e0b6c545899c856ef90bbfdb5f9de86f910978ca..94425db64883e59772fbfa30d93cd3cddd6f146d 100644 GIT binary patch delta 2750 zcmV;v3PJU)71R}fABzY86?7M000ZqDU325M@t*xFFz$nA+9M@emelG^wVl+{v#qh* z^dXbEfk;R~4H7(%lut?X-@6O&Ta+c+XE{l^4<`v9i^YCp0iRC;K4(!Jbw0cqbODCL zK7KmGez*LY!3U=|?4F&SI(-MmIj4Qc`9S(_M^XVv6jMci$Oqtol0R=Bx2ykm3Vtqs zpZP9zm&~(d$P|sGvRwh(VJc#3#a*_7Z(IiUn)>2dYoLG!gR?F2ce=gauqyx4-k>}D zK)P?V=L-Cv%b$k)2P4-J$>bi{u2f9ohu>K_RD?_F96j*B8ulR>C_{i#nhuQ z9UT!ugFwcAG=}B~e?o)sE%ufApNA{}E@9a_1E;ziWwT!Nnt=}K^5{Rg}#?Nj%s8U+IpK>4~*b0U^qfjW;9}hGwt`tvGEr^ z^Ei#yAxm4hu}8Ck%(65)w5Iqhh)XQqX0!SB%NUH5L&04NX6ck#BPNg;syq@%qp`dE zL}yGy)sF4HqKq?!7PeTY4j_Y=fw4hHe{HE!I4FZ?Tz#+gH}{SJqKGLEnA$*UL7xxF zjkmd)wQD5=7Oa|P8cP2?zCZuR=bQ2M<#aMWzuXuHmPJLvzEjfQM1nBqOn7%}-rSq^ zPqbNNytGSk^~Vz_K;x-eu=pnP;?c=ZTM3@rUTAU;Ln*Ggm6Y4b<=y!H_U=h~e|9PZ z0{7_xM&4acK7kNEUtN#C-$bYfO(3*RBv=Zl?^rz2b@Vb7L?|3<)e;cWcsX`mbj3GM zql8~RkD?-KB9BxeSjF8z6#ibxB&;a7mARWbNtH1b_lpfFdCiR-h~ZQ~JW|zu7I>lL z;HIj$F>*j67{n_n5}&z(a=$ilf5qyvz+C5i;nT2Af~#0%CZ`aP)$emL&A*h#xD1whDoUm5po>L&?)u+bQ!w_N>;Nx%7HFMrJloMD%;) z6l|1(=JQ}tI$G>zU8>b;#Z>YwG=Sorg)!Hte@(#PD-)vzGJE`&ce_J|%9ejJG zs&CWXW)pZRE!*<^CkQ+F^U2c-zn{%KBW4siSi#UD$Gpa**CeJ<5{6R6a0Vl>O3dme zDZPCR@`Pc>&?#7}U|UDeDY&B2@asgxd|%vSA&R^O+8@~ZQ7SJbU+c9Gd0i>DvrM#w zBsNhz1{Ua}9JGW=t~liJe`z?*k2L0#``+c1-#irczYVkcT%4^xyUKfh-7kZu>wn$8 z(;L+4f6k!aZ|i@DfO7p0zO%28)9IYle)wx-bJg>XXCGDrb1ga$;;{=?kZCRq0cBqUnOM0l9c!_OGz zyK)ggYLj8s)i7KVe~SZ^Ac^rRW(qPPbJY6?=soc3Ye0f)kR&K!(?cSgTtn6f6X$Y7 zmT?RjL8r6eai*hq)K5SW*BvYfC_YOtUF>*p1QDHxFD$CuCC3U;qF4tQXlcR&eVDd= z?@zg~%V~lMRbcj@HhHsF#jSitg+%$%bef^RU@tZySMDySfA@K=qD4^Ux#o5;Eh-p~ zi~>F|Wdt{egS*C<9n%{ z92-CF(lk_`8C%Q1`#i&oi>Q;*3FvjQv5bAOffXz0nfpYRe{YtYi^TY(HXHmbB#$HX zKE2agpV1@qOAA&#(|t)$Dz#H-xoE#l2~D`zt%iDBMG@r;j|4B(#2`BblwqeT-jDD9 zH8skpbrGql5wanSLPuY$lN&e-9es^XZlKjV`bIjr_f1G$A;B^F4VKgB{K{QaDZ1XN zq}AJ$MlG#1e>Gi`rETd6%;a^ITQs8~rSlyKN9nSxZh7zQwuN1gh1EBF{27 z2MAN??1pOVerVh!@Q(?z3mq0ef;!CvdoBhWaxR*PSR>jO1)i*HvWW0{#ih#IOBfn~ zxs=nhV=aRmN2iV7e|U)B`0X#k^WXnEgTZMV{|^CmlYfgfo-hC2;Iz&E4gz)gr_tb> zW`H~Lf4|fI*=d^p4V~d>7xTa2&}s9(Lx8zjnP&1o*Wyo7XGDtqOHef0L8l8k#XUC5`L4VdU6D`)|MT5c!9OJ?x)G5mWIiT)-~-2X|k8|Jic}t^FSY z_Ok!h;{UMFZ2$USeSG;AmB5|$U;qCfP@Ew^YyXFUvi(0Kp0wcF=4_lw&$@i!z z*PR{3+-EX@v#)d7JwhJF=lBRziiw~B%lpw3oW$JLJKXl&_WIV2Oofs9{^AQ E07lMTw*UYD delta 2712 zcmV;J3TO4y6|EJ2ABzY8$X*v;00ZqDU325M@t*xFFz$nA+9M@emelG^wVl+Hv#qh* z^dXbEfk;R~4H7(%lut?X-@6O&Ta+c+XSqqa4<`u(7K{DfC44>&_?$&?)cNpc&;=L{ z`}pY$``z+q1|OW>uzPlP>hv9$=bZK(=L6}#9Z3ZwQA`zoAs>JTO8&Ba-md4rQ}A>7 z`^doVKN`7?NGA8lcBNtxKm5+hp(0#T=jee4-iVwVm6#fpNS7Y@ET$ff z>F9_M8U!+brZEgh_!9<%Z?Uh`4+V=x`laOEYZ-g8d1RamavDQdCU0tE) z_+xn$5{L7zGbOG*&gg?F<;~zpKz)V_(=Dvr3XV(jeybphLgsEFiuf;xsJDF?FeQl^ zD3Bh*;UE#Bytw$9kN%c^`&oawG^8^wc+8Qnf2R?Dcga-%+h{y^un}o6>Q&e7+D;Fpo#T+VzblbqX4i zKM85P9FeFz^wsh zklc8iyIH$-LSVtFX{Dj`U*r4pe|)|fUtdlqPZs_trH2B0_r;!k8~ZqOa&1N$6B=ngfw1`T^C*P z&C@91m(Qc9h?>YFl?YaGcMyfYS277J3T|cYrcP32PR0G=KuTWoU^;KZ5bH4CtSSP_%tTGh>)2wzHm;!#LGUS1u zs||rcRDKH}vPw95!OBPydUeE)l}THLK*P#GwuqtR>8sT%0csauqc%lyIGg&X1aREvFit_vEjK=)PA*gil@B8Zs|8$Si?zbe1HCyPGAS$ zUa9Kabhp_AUP{ZhJpT#8PX2uI^v3UJGtY<_MGjUlw8$~9G3hmlX_SPaR58?GBvy&p z-6W;Ak3pU=?3g+QYZYwk>^TKjRGNOBh?wt-dn`nex4`%VTR%$WrQ~b9_93q;<#v{d zwvfaoipRhLUCO~osN{-69)F*PKh))#swN{_HC6`F+0(p6>s3 z`%Z6A>;E}}e!uPi9RkYzKlskRLQbc1Qv2brkcN)6FVs{MUBAz_~#H&<(ykV3Z_Q3M-YpPLLQg zhv?=UGh3$45on0a=DKAA^g!T-^FlbGX&a;fm_h?0Xa8nyY!mGJ0umA}Wgs`4BAhpRb>uwlsiGR(3N|3~O6*C2ykSX;(0(uYp`Wlen9wZ4$*z}ahCfATP!os;6 zk!2i1M$qXjc%11dmih@O;<|$k0mWwtri&d9N)XYB_`;&fU2^OYC5m-`ftDsb(8aXv zdwT}~5Br~T!6z&>hR;KA6uHYs$C;&9t=P^q9OM>nm}rig<4Krz+a4jgpSb! z70ZWQ$PWpfH{tZ9$LDLKoTyY>Ff#VMw9Q{~cO*d{N*PCpXY)9epF6-1|*PT_M3S`VE%T=={o$s8V#j zQ%S3DQyR6jHhh+lt@4Xxf{RaqB z>FS1V>wak5CGd|4a|#_6KY}{V1bZ$98*(n1h*%@q7X?<L&jddpuwMy}{|QmH#22F8?$deA5hY zNB(zuT7Q54#Gkcp$Nz^Uj>R|50-vA%oehU={(lf?mOoyuu<)hD zH_?RW%YQI*x~=>V0lVbi?c(R_>VX~kfA_T8uigLZ_J`dz|33tj?|-ebTVD7#Vw)Gb zKz|P`Y$}$cIt{}S2^Ku~nqTRv{!LEy*3itsC22yU2S$!PwExzI`S*i8?4L#vQ}HWY zz%Kg-cVEB%*>eW1{T~AMvj5iN|FF<(|N38Df_;ll;7)ZoAS4b(XRPyhg%2UV;9 diff --git a/internal/scorecard/testdata/bundle/tests/scorecard/config.yaml b/internal/scorecard/testdata/bundle/tests/scorecard/config.yaml index 4831301bbcc..7107497532a 100644 --- a/internal/scorecard/testdata/bundle/tests/scorecard/config.yaml +++ b/internal/scorecard/testdata/bundle/tests/scorecard/config.yaml @@ -1,45 +1,50 @@ -stages: -- parallel: true - tests: - - image: quay.io/operator-framework/scorecard-test:dev - entrypoint: - - scorecard-test - - basic-check-spec - labels: - suite: basic - test: basic-check-spec-test - - image: quay.io/operator-framework/scorecard-test:dev - entrypoint: - - scorecard-test - - olm-bundle-validation - labels: - suite: olm - test: olm-bundle-validation-test - - image: quay.io/operator-framework/scorecard-test:dev - entrypoint: - - scorecard-test - - olm-crds-have-validation - labels: - suite: olm - test: olm-crds-have-validation-test - - image: quay.io/operator-framework/scorecard-test:dev - entrypoint: - - scorecard-test - - olm-crds-have-resources - labels: - suite: olm - test: olm-crds-have-resources-test - - image: quay.io/operator-framework/scorecard-test:dev - entrypoint: - - scorecard-test - - olm-spec-descriptors - labels: - suite: olm - test: olm-spec-descriptors-test - - image: quay.io/operator-framework/scorecard-test:dev - entrypoint: - - scorecard-test - - olm-status-descriptors - labels: - suite: olm - test: olm-status-descriptors-test +kind: ScorecardConfiguration +apiversion: scorecard.operatorframework.io/v1alpha3 +metadata: + name: config +spec: + stages: + - parallel: true + tests: + - image: quay.io/operator-framework/scorecard-test:dev + entrypoint: + - scorecard-test + - basic-check-spec + labels: + suite: basic + test: basic-check-spec-test + - image: quay.io/operator-framework/scorecard-test:dev + entrypoint: + - scorecard-test + - olm-bundle-validation + labels: + suite: olm + test: olm-bundle-validation-test + - image: quay.io/operator-framework/scorecard-test:dev + entrypoint: + - scorecard-test + - olm-crds-have-validation + labels: + suite: olm + test: olm-crds-have-validation-test + - image: quay.io/operator-framework/scorecard-test:dev + entrypoint: + - scorecard-test + - olm-crds-have-resources + labels: + suite: olm + test: olm-crds-have-resources-test + - image: quay.io/operator-framework/scorecard-test:dev + entrypoint: + - scorecard-test + - olm-spec-descriptors + labels: + suite: olm + test: olm-spec-descriptors-test + - image: quay.io/operator-framework/scorecard-test:dev + entrypoint: + - scorecard-test + - olm-status-descriptors + labels: + suite: olm + test: olm-status-descriptors-test diff --git a/internal/scorecard/testpod.go b/internal/scorecard/testpod.go index 41203465fc7..3ccf5033e94 100644 --- a/internal/scorecard/testpod.go +++ b/internal/scorecard/testpod.go @@ -24,6 +24,8 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/rand" "k8s.io/client-go/kubernetes" + + "github.com/operator-framework/operator-sdk/pkg/apis/scorecard/v1alpha3" ) const ( @@ -33,7 +35,7 @@ const ( // getPodDefinition fills out a Pod definition based on // information from the test -func getPodDefinition(configMapName string, test Test, r PodTestRunner) *v1.Pod { +func getPodDefinition(configMapName string, test v1alpha3.TestConfiguration, r PodTestRunner) *v1.Pod { return &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("scorecard-test-%s", rand.String(4)), diff --git a/pkg/apis/scorecard/v1alpha3/configuration_types.go b/pkg/apis/scorecard/v1alpha3/configuration_types.go new file mode 100644 index 00000000000..c87cc85c0a6 --- /dev/null +++ b/pkg/apis/scorecard/v1alpha3/configuration_types.go @@ -0,0 +1,52 @@ +// Copyright 2020 The Operator-SDK Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1alpha3 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// scorecardConfigKind is the default scorecard componentconfig kind. +const ScorecardConfigurationKind = "ScorecardConfiguration" + +// ScorecardConfiguration is the Schema for the scorecardconfigurations API +type ScorecardConfiguration struct { + metav1.TypeMeta `json:",inline" yaml:",inline"` + // Do not use metav1.ObjectMeta because this object should not be apply-able. + Metadata struct { + Name string `json:"name,omitempty" yaml:"name,omitempty"` + } `json:"metadata,omitempty" yaml:"metadata,omitempty"` + + Spec ScorecardConfigurationSpec `json:"spec,omitempty" yaml:"spec,omitempty"` +} + +// ScorecardConfigurationSpec represents the set of test configurations which scorecard would run based on user input. +type ScorecardConfigurationSpec struct { + Stages []StageConfiguration `json:"stages" yaml:"stages"` +} + +type StageConfiguration struct { + Parallel bool `json:"parallel,omitempty" yaml:"parallel,omitempty"` + Tests []TestConfiguration `json:"tests" yaml:"tests"` +} + +type TestConfiguration struct { + // Image is the name of the testimage + Image string `json:"image" yaml:"image"` + // Entrypoint is list of commands and arguments passed to the test image + Entrypoint []string `json:"entrypoint,omitempty" yaml:"entrypoint,omitempty"` + // Labels that further describe the test and enable selection + Labels map[string]string `json:"labels,omitempty" yaml:"labels,omitempty"` +} diff --git a/pkg/apis/scorecard/v1alpha3/types.go b/pkg/apis/scorecard/v1alpha3/test_types.go similarity index 100% rename from pkg/apis/scorecard/v1alpha3/types.go rename to pkg/apis/scorecard/v1alpha3/test_types.go diff --git a/website/content/en/docs/advanced-topics/scorecard/scorecard.md b/website/content/en/docs/advanced-topics/scorecard/scorecard.md index 532d5bbda91..1005c633717 100644 --- a/website/content/en/docs/advanced-topics/scorecard/scorecard.md +++ b/website/content/en/docs/advanced-topics/scorecard/scorecard.md @@ -30,18 +30,14 @@ require if the tests are designed for resource creation. ## Running the Scorecard +1. Define a scorecard configuration file by running + ```sh + $ operator-sdk generate kustomize scorecard + ``` + See the [config file section](#config-file) for an explanation of the configuration file format. 1. Generate [bundle][quickstart-bundle] manifests and metadata for your Operator. `make bundle` will automatically add scorecard annotations to your bundle's metadata, which is used by the `scorecard` command to run tests. -1. Define a scorecard configuration file `bundle/tests/scorecard/config.yaml`, the default path. -If you choose to define this file elsewhere, you can either change the modified portion of that path in -`annotations.yaml` and `bundle.Dockerfile`, or override that value using the `--config` flag. -Unless executing custom tests, you may copy this [sample][sample-config] configuration file into your project. -See the [config file section](#config-file) for an explanation of the configuration file format. -1. Add the following line to the end of your `bundle.Dockerfile`: - ```docker - COPY bundle/tests/scorecard /tests/scorecard - ``` 1. Execute the [`scorecard` command][cli-scorecard]. See the [command args section](#command-args) for an overview of command invocation. diff --git a/website/content/en/docs/cli/operator-sdk_generate_kustomize.md b/website/content/en/docs/cli/operator-sdk_generate_kustomize.md index 2e529b7d146..5ac1e28ffa5 100644 --- a/website/content/en/docs/cli/operator-sdk_generate_kustomize.md +++ b/website/content/en/docs/cli/operator-sdk_generate_kustomize.md @@ -25,4 +25,5 @@ Contains subcommands that generate operator-framework kustomize data for the ope * [operator-sdk generate](../operator-sdk_generate) - Invokes a specific generator * [operator-sdk generate kustomize manifests](../operator-sdk_generate_kustomize_manifests) - Generates kustomize bases and a kustomization.yaml for operator-framework manifests +* [operator-sdk generate kustomize scorecard](../operator-sdk_generate_kustomize_scorecard) - Generates scorecard configuration files diff --git a/website/content/en/docs/cli/operator-sdk_generate_kustomize_scorecard.md b/website/content/en/docs/cli/operator-sdk_generate_kustomize_scorecard.md new file mode 100644 index 00000000000..e9f246d4b24 --- /dev/null +++ b/website/content/en/docs/cli/operator-sdk_generate_kustomize_scorecard.md @@ -0,0 +1,56 @@ +--- +title: "operator-sdk generate kustomize scorecard" +--- +## operator-sdk generate kustomize scorecard + +Generates scorecard configuration files + +### Synopsis + + +Running 'generate kustomize scorecard' will (re)generate scorecard configuration kustomize bases, +default test patches, and a kustomization.yaml in 'config/scorecard'. + + +``` +operator-sdk generate kustomize scorecard [flags] +``` + +### Examples + +``` + + $ operator-sdk generate kustomize scorecard + Generating kustomize files in config/scorecard + Kustomize files generated successfully + $ tree ./config/scorecard + ./config/scorecard/ + ├── bases + │   └── config.yaml + ├── kustomization.yaml + └── patches + ├── basic.config.yaml + └── olm.config.yaml + +``` + +### Options + +``` + -h, --help help for scorecard + --image /scorecard-test Image to use for default tests; this image must contain the /scorecard-test binary (default "quay.io/operator-framework/scorecard-test:latest") + --operator-name string Name of the operator + --output-dir string Directory to write kustomize files + -q, --quiet Run in quiet mode +``` + +### Options inherited from parent commands + +``` + --verbose Enable verbose logging +``` + +### SEE ALSO + +* [operator-sdk generate kustomize](../operator-sdk_generate_kustomize) - Contains subcommands that generate operator-framework kustomize data for the operator + From ff1c2b0d4be9f42f080be2a4d6abdeee25011f21 Mon Sep 17 00:00:00 2001 From: Eric Stroczynski Date: Wed, 22 Jul 2020 14:13:15 -0700 Subject: [PATCH 2/7] move config/scorecard scaffold to 'init' plugins --- cmd/operator-sdk/generate/kustomize/cmd.go | 3 - .../generate/kustomize/manifests.go | 14 +- .../generate/kustomize/scorecard/cmd.go | 133 ------------------ internal/plugins/golang/v2/init.go | 15 ++ internal/plugins/helm/v1/init.go | 18 ++- .../plugins/scorecard/init.go | 41 ++++-- .../cli/operator-sdk_generate_kustomize.md | 1 - ...erator-sdk_generate_kustomize_scorecard.md | 56 -------- 8 files changed, 61 insertions(+), 220 deletions(-) delete mode 100644 cmd/operator-sdk/generate/kustomize/scorecard/cmd.go rename cmd/operator-sdk/generate/kustomize/scorecard/kustomize.go => internal/plugins/scorecard/init.go (86%) delete mode 100644 website/content/en/docs/cli/operator-sdk_generate_kustomize_scorecard.md diff --git a/cmd/operator-sdk/generate/kustomize/cmd.go b/cmd/operator-sdk/generate/kustomize/cmd.go index 6aca12f275e..30ae4f00f57 100644 --- a/cmd/operator-sdk/generate/kustomize/cmd.go +++ b/cmd/operator-sdk/generate/kustomize/cmd.go @@ -16,8 +16,6 @@ package kustomize import ( "github.com/spf13/cobra" - - "github.com/operator-framework/operator-sdk/cmd/operator-sdk/generate/kustomize/scorecard" ) // NewCmd returns the 'kustomize' subcommand. @@ -29,7 +27,6 @@ func NewCmd() *cobra.Command { cmd.AddCommand( newManifestsCmd(), - scorecard.NewCmd(), ) return cmd diff --git a/cmd/operator-sdk/generate/kustomize/manifests.go b/cmd/operator-sdk/generate/kustomize/manifests.go index 42991edd815..53e2e21f48f 100644 --- a/cmd/operator-sdk/generate/kustomize/manifests.go +++ b/cmd/operator-sdk/generate/kustomize/manifests.go @@ -16,7 +16,6 @@ package kustomize import ( "fmt" - "os" "path/filepath" log "github.com/sirupsen/logrus" @@ -154,6 +153,7 @@ func (c *manifestsCmd) setDefaults(cfg *config.Config) { const manifestsKustomization = `resources: - ../default - ../samples +- ../scorecard ` // run generates kustomize bundle bases and a kustomization.yaml if one does not exist. @@ -175,18 +175,8 @@ func (c manifestsCmd) run(cfg *config.Config) error { return fmt.Errorf("error generating kustomize bases: %v", err) } - // NB(estroz): this is a rather hacky way of adding scorecard componentconfigs to the bundle, - // and won't work if the manifests kustomization.yaml already exists. This should be improved - // with scaffolding markers. - kustomization := manifestsKustomization - - // Add a scorecard kustomization if one exists. - info, err := os.Stat(filepath.Join(filepath.Dir(c.outputDir), "scorecard")) - if err == nil && info.IsDir() { - kustomization += "- ../scorecard\n" - } // Write a kustomization.yaml to outputDir if one does not exist. - if err := kustomize.WriteIfNotExist(c.outputDir, kustomization); err != nil { + if err := kustomize.WriteIfNotExist(c.outputDir, manifestsKustomization); err != nil { return fmt.Errorf("error writing kustomization.yaml: %v", err) } diff --git a/cmd/operator-sdk/generate/kustomize/scorecard/cmd.go b/cmd/operator-sdk/generate/kustomize/scorecard/cmd.go deleted file mode 100644 index d8269034317..00000000000 --- a/cmd/operator-sdk/generate/kustomize/scorecard/cmd.go +++ /dev/null @@ -1,133 +0,0 @@ -// Copyright 2020 The Operator-SDK Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package scorecard - -import ( - "fmt" - "path/filepath" - - log "github.com/sirupsen/logrus" - "github.com/spf13/cobra" - "github.com/spf13/pflag" - "sigs.k8s.io/kubebuilder/pkg/model/config" - - kbutil "github.com/operator-framework/operator-sdk/internal/util/kubebuilder" -) - -const scorecardLongHelp = ` -Running 'generate kustomize scorecard' will (re)generate scorecard configuration kustomize bases, -default test patches, and a kustomization.yaml in 'config/scorecard'. -` - -const scorecardExamples = ` - $ operator-sdk generate kustomize scorecard - Generating kustomize files in config/scorecard - Kustomize files generated successfully - $ tree ./config/scorecard - ./config/scorecard/ - ├── bases - │   └── config.yaml - ├── kustomization.yaml - └── patches - ├── basic.config.yaml - └── olm.config.yaml -` - -// defaultTestImageTag points to the latest-released image. -// TODO: change the tag to "latest" once config scaffolding is in a release, -// as the new config spec won't work with the current latest image. -const defaultTestImageTag = "quay.io/operator-framework/scorecard-test:master" - -type scorecardCmd struct { - operatorName string - outputDir string - testImageTag string - quiet bool -} - -// NewCmd returns the `scorecard` subcommand. -func NewCmd() *cobra.Command { - c := &scorecardCmd{} - cmd := &cobra.Command{ - Use: "scorecard", - Short: "Generates scorecard configuration files", - Long: scorecardLongHelp, - Example: scorecardExamples, - RunE: func(cmd *cobra.Command, args []string) error { - if len(args) != 0 { - return fmt.Errorf("command %s doesn't accept any arguments", cmd.CommandPath()) - } - - cfg, err := kbutil.ReadConfig() - if err != nil { - return fmt.Errorf("error reading configuration: %v", err) - } - c.setDefaults(cfg) - - // Run command logic. - if err = c.run(); err != nil { - log.Fatalf("Error generating kustomize files: %v", err) - } - - return nil - }, - } - - c.addFlagsTo(cmd.Flags()) - - return cmd -} - -func (c *scorecardCmd) addFlagsTo(fs *pflag.FlagSet) { - fs.StringVar(&c.operatorName, "operator-name", "", "Name of the operator") - fs.StringVar(&c.outputDir, "output-dir", "", "Directory to write kustomize files") - fs.StringVar(&c.testImageTag, "image", defaultTestImageTag, - "Image to use for default tests; this image must contain the `/scorecard-test` binary") - fs.BoolVarP(&c.quiet, "quiet", "q", false, "Run in quiet mode") - // NB(estroz): might be nice to have an --overwrite flag to explicitly turn on overwrite behavior (the current default). -} - -// defaultDir is the default directory in which to generate kustomize bases and the kustomization.yaml. -var defaultDir = filepath.Join("config", "scorecard") - -// setDefaults sets command defaults. -func (c *scorecardCmd) setDefaults(cfg *config.Config) { - if c.operatorName == "" { - c.operatorName = filepath.Base(cfg.Repo) - } - - if c.outputDir == "" { - c.outputDir = defaultDir - } -} - -// run scaffolds kustomize files for kustomizing a scorecard componentconfig. -func (c scorecardCmd) run() error { - - if !c.quiet { - fmt.Println("Generating kustomize files in", c.outputDir) - } - - err := generate(c.operatorName, c.testImageTag, c.outputDir) - if err != nil { - return err - } - - if !c.quiet { - fmt.Println("Kustomize files generated successfully") - } - - return nil -} diff --git a/internal/plugins/golang/v2/init.go b/internal/plugins/golang/v2/init.go index 1e00261d558..4cc5c3511dc 100644 --- a/internal/plugins/golang/v2/init.go +++ b/internal/plugins/golang/v2/init.go @@ -16,11 +16,15 @@ package v2 import ( "fmt" + "os" + "path/filepath" + "strings" "github.com/spf13/pflag" "sigs.k8s.io/kubebuilder/pkg/model/config" "sigs.k8s.io/kubebuilder/pkg/plugin" + "github.com/operator-framework/operator-sdk/internal/plugins/scorecard" utilplugins "github.com/operator-framework/operator-sdk/internal/util/plugins" ) @@ -50,6 +54,17 @@ func (p *initPlugin) Run() error { return err } + // Assume projectName was validated by go.kubebuilder.io. + wd, err := os.Getwd() + if err != nil { + return fmt.Errorf("error getting the current path: %v", err) + } + projectName := strings.ToLower(filepath.Base(wd)) + // Run the scorecard "phase 2" plugin. + if err := scorecard.RunInit(projectName); err != nil { + return err + } + // Update plugin config section with this plugin's configuration. cfg := Config{} if err := p.config.EncodePluginConfig(pluginConfigKey, cfg); err != nil { diff --git a/internal/plugins/helm/v1/init.go b/internal/plugins/helm/v1/init.go index eddd889ad4b..352032a6cf1 100644 --- a/internal/plugins/helm/v1/init.go +++ b/internal/plugins/helm/v1/init.go @@ -29,6 +29,7 @@ import ( "github.com/operator-framework/operator-sdk/internal/kubebuilder/cmdutil" "github.com/operator-framework/operator-sdk/internal/plugins/helm/v1/chartutil" "github.com/operator-framework/operator-sdk/internal/plugins/helm/v1/scaffolds" + "github.com/operator-framework/operator-sdk/internal/plugins/scorecard" utilplugins "github.com/operator-framework/operator-sdk/internal/util/plugins" ) @@ -128,7 +129,22 @@ func (p *initPlugin) InjectConfig(c *config.Config) { // Run will call the plugin actions func (p *initPlugin) Run() error { - return cmdutil.Run(p) + if err := cmdutil.Run(p); err != nil { + return err + } + + // Assume projectName was validated by Validate(). + wd, err := os.Getwd() + if err != nil { + return fmt.Errorf("error getting the current path: %v", err) + } + projectName := strings.ToLower(filepath.Base(wd)) + // Run the scorecard "phase 2" plugin. + if err := scorecard.RunInit(projectName); err != nil { + return err + } + + return nil } // Validate perform the required validations for this plugin diff --git a/cmd/operator-sdk/generate/kustomize/scorecard/kustomize.go b/internal/plugins/scorecard/init.go similarity index 86% rename from cmd/operator-sdk/generate/kustomize/scorecard/kustomize.go rename to internal/plugins/scorecard/init.go index 07672c32fb3..26e4731e43f 100644 --- a/cmd/operator-sdk/generate/kustomize/scorecard/kustomize.go +++ b/internal/plugins/scorecard/init.go @@ -47,8 +47,28 @@ patchesJson6902: {{- end }} ` -// scorecardKustomization holds data required to generate a scorecard's kustomization.yaml. -type scorecardKustomization struct { +const ( + // defaultTestImageTag points to the latest-released image. + // TODO: change the tag to "latest" once config scaffolding is in a release, + // as the new config spec won't work with the current latest image. + defaultTestImageTag = "quay.io/operator-framework/scorecard-test:master" + + // scorecardConfigName is the default scorecard componentconfig's metadata.name, + // which must be set on all kustomize-able bases. This name is only used for + // `kustomize build` pattern match and not for on-cluster creation. + scorecardConfigName = "config" +) + +// defaultDir is the default directory in which to generate kustomize bases and the kustomization.yaml. +var defaultDir = filepath.Join("config", "scorecard") + +// RunInit scaffolds kustomize files for kustomizing a scorecard componentconfig. +func RunInit(projectName string) error { + return generate(projectName, defaultTestImageTag, defaultDir) +} + +// scorecardKustomizationValues holds data required to generate a scorecard's kustomization.yaml. +type scorecardKustomizationValues struct { ResourcePaths []string JSONPatches []kustomizationJSON6902Patch } @@ -59,18 +79,11 @@ type kustomizationJSON6902Patch struct { Target registry.DefinitionKey } -const ( - // scorecardConfigName is the default scorecard componentconfig's metadata.name, - // which must be set on all kustomize-able bases. This name is only used for - // `kustomize build` pattern match and not for "creation". - scorecardConfigName = "config" -) - // generate scaffolds kustomize bundle bases and a kustomization.yaml. // TODO(estroz): refactor this to be testable (in-mem fs) and easier to read. func generate(operatorName, testImageTag, outputDir string) error { - kustomization := scorecardKustomization{} + kustomizationValues := scorecardKustomizationValues{} // Config bases. basesDir := filepath.Join(outputDir, "bases") @@ -88,7 +101,7 @@ func generate(operatorName, testImageTag, outputDir string) error { if err := ioutil.WriteFile(basePath, b, 0666); err != nil { return fmt.Errorf("error writing default scorecard config: %v", err) } - kustomization.ResourcePaths = append(kustomization.ResourcePaths, relBasePath) + kustomizationValues.ResourcePaths = append(kustomizationValues.ResourcePaths, relBasePath) scorecardConfigTarget := registry.DefinitionKey{ Group: v1alpha3.SchemeGroupVersion.Group, Version: v1alpha3.SchemeGroupVersion.Version, @@ -112,7 +125,7 @@ func generate(operatorName, testImageTag, outputDir string) error { if err := ioutil.WriteFile(filepath.Join(patchesDir, basicPatchFileName), b, 0666); err != nil { return fmt.Errorf("error writing basic scorecard config patch: %v", err) } - kustomization.JSONPatches = append(kustomization.JSONPatches, kustomizationJSON6902Patch{ + kustomizationValues.JSONPatches = append(kustomizationValues.JSONPatches, kustomizationJSON6902Patch{ Path: filepath.Join("patches", basicPatchFileName), Target: scorecardConfigTarget, }) @@ -127,7 +140,7 @@ func generate(operatorName, testImageTag, outputDir string) error { if err := ioutil.WriteFile(filepath.Join(patchesDir, olmPatchFileName), b, 0666); err != nil { return fmt.Errorf("error writing default scorecard config: %v", err) } - kustomization.JSONPatches = append(kustomization.JSONPatches, kustomizationJSON6902Patch{ + kustomizationValues.JSONPatches = append(kustomizationValues.JSONPatches, kustomizationJSON6902Patch{ Path: filepath.Join("patches", olmPatchFileName), Target: scorecardConfigTarget, }) @@ -138,7 +151,7 @@ func generate(operatorName, testImageTag, outputDir string) error { return fmt.Errorf("error parsing default kustomize template: %v", err) } buf := bytes.Buffer{} - if err = t.Execute(&buf, kustomization); err != nil { + if err = t.Execute(&buf, kustomizationValues); err != nil { return fmt.Errorf("error executing on default kustomize template: %v", err) } if err := kustomize.Write(outputDir, buf.String()); err != nil { diff --git a/website/content/en/docs/cli/operator-sdk_generate_kustomize.md b/website/content/en/docs/cli/operator-sdk_generate_kustomize.md index 5ac1e28ffa5..2e529b7d146 100644 --- a/website/content/en/docs/cli/operator-sdk_generate_kustomize.md +++ b/website/content/en/docs/cli/operator-sdk_generate_kustomize.md @@ -25,5 +25,4 @@ Contains subcommands that generate operator-framework kustomize data for the ope * [operator-sdk generate](../operator-sdk_generate) - Invokes a specific generator * [operator-sdk generate kustomize manifests](../operator-sdk_generate_kustomize_manifests) - Generates kustomize bases and a kustomization.yaml for operator-framework manifests -* [operator-sdk generate kustomize scorecard](../operator-sdk_generate_kustomize_scorecard) - Generates scorecard configuration files diff --git a/website/content/en/docs/cli/operator-sdk_generate_kustomize_scorecard.md b/website/content/en/docs/cli/operator-sdk_generate_kustomize_scorecard.md deleted file mode 100644 index e9f246d4b24..00000000000 --- a/website/content/en/docs/cli/operator-sdk_generate_kustomize_scorecard.md +++ /dev/null @@ -1,56 +0,0 @@ ---- -title: "operator-sdk generate kustomize scorecard" ---- -## operator-sdk generate kustomize scorecard - -Generates scorecard configuration files - -### Synopsis - - -Running 'generate kustomize scorecard' will (re)generate scorecard configuration kustomize bases, -default test patches, and a kustomization.yaml in 'config/scorecard'. - - -``` -operator-sdk generate kustomize scorecard [flags] -``` - -### Examples - -``` - - $ operator-sdk generate kustomize scorecard - Generating kustomize files in config/scorecard - Kustomize files generated successfully - $ tree ./config/scorecard - ./config/scorecard/ - ├── bases - │   └── config.yaml - ├── kustomization.yaml - └── patches - ├── basic.config.yaml - └── olm.config.yaml - -``` - -### Options - -``` - -h, --help help for scorecard - --image /scorecard-test Image to use for default tests; this image must contain the /scorecard-test binary (default "quay.io/operator-framework/scorecard-test:latest") - --operator-name string Name of the operator - --output-dir string Directory to write kustomize files - -q, --quiet Run in quiet mode -``` - -### Options inherited from parent commands - -``` - --verbose Enable verbose logging -``` - -### SEE ALSO - -* [operator-sdk generate kustomize](../operator-sdk_generate_kustomize) - Contains subcommands that generate operator-framework kustomize data for the operator - From 3187148f4fbc24bb931b8446a796a0a264fe36a2 Mon Sep 17 00:00:00 2001 From: Eric Stroczynski Date: Wed, 22 Jul 2020 14:27:15 -0700 Subject: [PATCH 3/7] add changelog fragment --- changelog/fragments/scorecard-config-scaffold.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 changelog/fragments/scorecard-config-scaffold.yaml diff --git a/changelog/fragments/scorecard-config-scaffold.yaml b/changelog/fragments/scorecard-config-scaffold.yaml new file mode 100644 index 00000000000..38038e0d18b --- /dev/null +++ b/changelog/fragments/scorecard-config-scaffold.yaml @@ -0,0 +1,9 @@ +entries: + - description: Implemented the scorecard config as a componentconfig object + kind: change + breaking: true + migration: + header: Update your scorecard config file to the new format + body: See the updated scorecard [config documentation](https://sdk.operatorframework.io/docs/scorecard/scorecard/#config-file) for details. + - description: Added `config/scorecard` kustomize scaffolds to `init` + kind: addition From 22883b943170a3b8ca5925e436e987a9a0bdf957 Mon Sep 17 00:00:00 2001 From: Eric Stroczynski Date: Wed, 22 Jul 2020 14:57:25 -0700 Subject: [PATCH 4/7] updates to config spec --- cmd/operator-sdk/generate/bundle/bundle.go | 2 +- internal/generate/collector/collect.go | 8 +- internal/plugins/scorecard/init.go | 10 +- internal/scorecard/config.go | 4 +- .../bundle/tests/scorecard/config.yaml | 33 +++-- internal/scorecard/formatting.go | 2 +- internal/scorecard/labels_test.go | 132 +++++++++--------- internal/scorecard/run_test.go | 16 +-- internal/scorecard/scorecard.go | 4 +- internal/scorecard/testdata/bundle.tar.gz | Bin 2772 -> 2768 bytes .../bundle/tests/scorecard/config.yaml | 91 ++++++------ .../scorecard/v1alpha3/configuration_types.go | 32 +++-- 12 files changed, 165 insertions(+), 169 deletions(-) diff --git a/cmd/operator-sdk/generate/bundle/bundle.go b/cmd/operator-sdk/generate/bundle/bundle.go index 7a0e5bd9759..c9cb0117f13 100644 --- a/cmd/operator-sdk/generate/bundle/bundle.go +++ b/cmd/operator-sdk/generate/bundle/bundle.go @@ -232,7 +232,7 @@ func (c bundleCmd) runManifests(cfg *config.Config) (err error) { } // writeScorecardConfig writes cfg to dir at the hard-coded config path 'config.yaml'. -func writeScorecardConfig(dir string, cfg v1alpha3.ScorecardConfiguration) error { +func writeScorecardConfig(dir string, cfg v1alpha3.Configuration) error { if cfg.Metadata.Name == "" { return nil } diff --git a/internal/generate/collector/collect.go b/internal/generate/collector/collect.go index 61d7c4cb319..35f2c5f8381 100644 --- a/internal/generate/collector/collect.go +++ b/internal/generate/collector/collect.go @@ -46,7 +46,7 @@ type Manifests struct { ValidatingWebhooks []admissionregv1.ValidatingWebhook MutatingWebhooks []admissionregv1.MutatingWebhook CustomResources []unstructured.Unstructured - ScorecardConfig scorecardv1alpha3.ScorecardConfiguration + ScorecardConfig scorecardv1alpha3.Configuration Others []unstructured.Unstructured } @@ -89,7 +89,7 @@ func (c *Manifests) UpdateFromDirs(deployDir, crdsDir string) error { err = c.addValidatingWebhookConfigurations(manifest) case "MutatingWebhookConfiguration": err = c.addMutatingWebhookConfigurations(manifest) - case scorecardv1alpha3.ScorecardConfigurationKind: + case scorecardv1alpha3.ConfigurationKind: err = c.addScorecardConfig(manifest) default: err = c.addOthers(manifest) @@ -151,7 +151,7 @@ func (c *Manifests) UpdateFromReader(r io.Reader) error { err = c.addValidatingWebhookConfigurations(manifest) case "MutatingWebhookConfiguration": err = c.addMutatingWebhookConfigurations(manifest) - case scorecardv1alpha3.ScorecardConfigurationKind: + case scorecardv1alpha3.ConfigurationKind: err = c.addScorecardConfig(manifest) default: err = c.addOthers(manifest) @@ -268,7 +268,7 @@ func (c *Manifests) addMutatingWebhookConfigurations(rawManifests ...[]byte) err // addScorecardConfig assumes manifest data in rawManifests is a ScorecardConfigs and adds it to the collector. // If a config has already been found, addScorecardConfig will return an error. func (c *Manifests) addScorecardConfig(rawManifest []byte) error { - cfg := scorecardv1alpha3.ScorecardConfiguration{} + cfg := scorecardv1alpha3.Configuration{} if err := yaml.Unmarshal(rawManifest, &cfg); err != nil { return err } diff --git a/internal/plugins/scorecard/init.go b/internal/plugins/scorecard/init.go index 26e4731e43f..1588d95d7a5 100644 --- a/internal/plugins/scorecard/init.go +++ b/internal/plugins/scorecard/init.go @@ -105,7 +105,7 @@ func generate(operatorName, testImageTag, outputDir string) error { scorecardConfigTarget := registry.DefinitionKey{ Group: v1alpha3.SchemeGroupVersion.Group, Version: v1alpha3.SchemeGroupVersion.Version, - Kind: v1alpha3.ScorecardConfigurationKind, + Kind: v1alpha3.ConfigurationKind, Name: scorecardConfigName, } @@ -174,10 +174,10 @@ type jsonPatchObject struct { // newScorecardConfigurationBase returns a scorecard componentconfig object with one parallel stage. // The returned object is intended to be marshaled and written to disk as a kustomize base. -func newScorecardConfigurationBase() (cfg v1alpha3.ScorecardConfiguration) { - cfg.SetGroupVersionKind(v1alpha3.SchemeGroupVersion.WithKind(v1alpha3.ScorecardConfigurationKind)) +func newScorecardConfigurationBase() (cfg v1alpha3.Configuration) { + cfg.SetGroupVersionKind(v1alpha3.SchemeGroupVersion.WithKind(v1alpha3.ConfigurationKind)) cfg.Metadata.Name = scorecardConfigName - cfg.Spec.Stages = []v1alpha3.StageConfiguration{ + cfg.Stages = []v1alpha3.StageConfiguration{ { Parallel: true, Tests: []v1alpha3.TestConfiguration{}, @@ -187,7 +187,7 @@ func newScorecardConfigurationBase() (cfg v1alpha3.ScorecardConfiguration) { } func makeTestStageJSONPath(stageIdx, testIdx int) string { - return fmt.Sprintf("/spec/stages/%d/tests/%d", stageIdx, testIdx) + return fmt.Sprintf("/stages/%d/tests/%d", stageIdx, testIdx) } // newBasicScorecardConfigurationPatch returns default "basic" test configurations as JSON patch objects diff --git a/internal/scorecard/config.go b/internal/scorecard/config.go index f87bf94939e..cf1a74fa336 100644 --- a/internal/scorecard/config.go +++ b/internal/scorecard/config.go @@ -35,8 +35,8 @@ const ( // location: tests/scorecard/config.yaml // the user can override this location using the --config CLI flag // TODO: version this. -func LoadConfig(configFilePath string) (v1alpha3.ScorecardConfiguration, error) { - c := v1alpha3.ScorecardConfiguration{} +func LoadConfig(configFilePath string) (v1alpha3.Configuration, error) { + c := v1alpha3.Configuration{} yamlFile, err := ioutil.ReadFile(configFilePath) if err != nil { diff --git a/internal/scorecard/examples/custom-scorecard-tests/bundle/tests/scorecard/config.yaml b/internal/scorecard/examples/custom-scorecard-tests/bundle/tests/scorecard/config.yaml index 944f89d6a5a..e1cc333f526 100644 --- a/internal/scorecard/examples/custom-scorecard-tests/bundle/tests/scorecard/config.yaml +++ b/internal/scorecard/examples/custom-scorecard-tests/bundle/tests/scorecard/config.yaml @@ -2,20 +2,19 @@ kind: ScorecardConfiguration apiversion: scorecard.operatorframework.io/v1alpha3 metadata: name: config -spec: - stages: - - tests: - - image: quay.io/username/custom-scorecard-tests:dev - entrypoint: - - custom-scorecard-tests - - customtest1 - labels: - suite: custom - test: customtest1 - - image: quay.io/username/custom-scorecard-tests:dev - entrypoint: - - custom-scorecard-tests - - customtest2 - labels: - suite: custom - test: customtest2 +stages: +- tests: + - image: quay.io/username/custom-scorecard-tests:dev + entrypoint: + - custom-scorecard-tests + - customtest1 + labels: + suite: custom + test: customtest1 + - image: quay.io/username/custom-scorecard-tests:dev + entrypoint: + - custom-scorecard-tests + - customtest2 + labels: + suite: custom + test: customtest2 diff --git a/internal/scorecard/formatting.go b/internal/scorecard/formatting.go index b32f3f45361..6e2778cc8c1 100644 --- a/internal/scorecard/formatting.go +++ b/internal/scorecard/formatting.go @@ -42,7 +42,7 @@ func (r PodTestRunner) getTestStatus(ctx context.Context, p *v1.Pod) (output *v1 // run based on user selection func (o Scorecard) List() v1alpha3.TestList { output := v1alpha3.NewTestList() - for _, stage := range o.Config.Spec.Stages { + for _, stage := range o.Config.Stages { tests := o.selectTests(stage) for _, test := range tests { item := v1alpha3.NewTest() diff --git a/internal/scorecard/labels_test.go b/internal/scorecard/labels_test.go index 6982cf9bbcb..6c6e31035b8 100644 --- a/internal/scorecard/labels_test.go +++ b/internal/scorecard/labels_test.go @@ -27,7 +27,7 @@ func TestEmptySelector(t *testing.T) { cases := []struct { selectorValue string testsSelected int - config v1alpha3.ScorecardConfiguration + config v1alpha3.Configuration wantError bool }{ {"", 7, testConfig, false}, @@ -53,7 +53,7 @@ func TestEmptySelector(t *testing.T) { return } - tests := o.selectTests(o.Config.Spec.Stages[0]) + tests := o.selectTests(o.Config.Stages[0]) testsSelected := len(tests) if testsSelected != c.testsSelected { t.Errorf("Wanted testsSelected %d, got: %d", c.testsSelected, testsSelected) @@ -63,82 +63,80 @@ func TestEmptySelector(t *testing.T) { } } -var testConfig = v1alpha3.ScorecardConfiguration{ - Spec: v1alpha3.ScorecardConfigurationSpec{ - Stages: []v1alpha3.StageConfiguration{ - { - Tests: []v1alpha3.TestConfiguration{ - {Image: "quay.io/someuser/customtest1:v0.0.1", - Entrypoint: []string{ - "custom-test", - }, - Labels: map[string]string{ - "suite": "custom", - "test": "customtest1", - }, +var testConfig = v1alpha3.Configuration{ + Stages: []v1alpha3.StageConfiguration{ + { + Tests: []v1alpha3.TestConfiguration{ + {Image: "quay.io/someuser/customtest1:v0.0.1", + Entrypoint: []string{ + "custom-test", }, + Labels: map[string]string{ + "suite": "custom", + "test": "customtest1", + }, + }, - {Image: "quay.io/someuser/customtest2:v0.0.1", - Entrypoint: []string{ - "custom-test", - }, - Labels: map[string]string{ - "suite": "custom", - "test": "customtest2", - }, + {Image: "quay.io/someuser/customtest2:v0.0.1", + Entrypoint: []string{ + "custom-test", }, + Labels: map[string]string{ + "suite": "custom", + "test": "customtest2", + }, + }, - {Image: "quay.io/redhat/basictests:v0.0.1", - Entrypoint: []string{ - "scorecard-test", - "basic-check-spec", - }, - Labels: map[string]string{ - "suite": "basic", - "test": "basic-check-spec-test", - }, + {Image: "quay.io/redhat/basictests:v0.0.1", + Entrypoint: []string{ + "scorecard-test", + "basic-check-spec", }, + Labels: map[string]string{ + "suite": "basic", + "test": "basic-check-spec-test", + }, + }, - {Image: "quay.io/redhat/basictests:v0.0.1", - Entrypoint: []string{ - "scorecard-test", - "basic-check-status", - }, - Labels: map[string]string{ - "suite": "basic", - "test": "basic-check-status-test", - }, + {Image: "quay.io/redhat/basictests:v0.0.1", + Entrypoint: []string{ + "scorecard-test", + "basic-check-status", + }, + Labels: map[string]string{ + "suite": "basic", + "test": "basic-check-status-test", }, + }, - {Image: "quay.io/redhat/olmtests:v0.0.1", - Entrypoint: []string{ - "scorecard-test", - "olm-bundle-validation", - }, - Labels: map[string]string{ - "suite": "olm", - "test": "olm-bundle-validation-test", - }, + {Image: "quay.io/redhat/olmtests:v0.0.1", + Entrypoint: []string{ + "scorecard-test", + "olm-bundle-validation", + }, + Labels: map[string]string{ + "suite": "olm", + "test": "olm-bundle-validation-test", }, + }, - {Image: "quay.io/redhat/olmtests:v0.0.1", - Entrypoint: []string{ - "scorecard-test", - "olm-crds-have-validation", - }, - Labels: map[string]string{ - "suite": "olm", - "test": "olm-crds-have-validation-test", - }, + {Image: "quay.io/redhat/olmtests:v0.0.1", + Entrypoint: []string{ + "scorecard-test", + "olm-crds-have-validation", + }, + Labels: map[string]string{ + "suite": "olm", + "test": "olm-crds-have-validation-test", + }, + }, + {Image: "quay.io/redhat/kuttltests:v0.0.1", + Entrypoint: []string{ + "kuttl-test", + "olm-status-descriptors", }, - {Image: "quay.io/redhat/kuttltests:v0.0.1", - Entrypoint: []string{ - "kuttl-test", - "olm-status-descriptors", - }, - Labels: map[string]string{ - "suite": "kuttl", - }, + Labels: map[string]string{ + "suite": "kuttl", }, }, }, diff --git a/internal/scorecard/run_test.go b/internal/scorecard/run_test.go index 708579f5a9d..52648b64156 100644 --- a/internal/scorecard/run_test.go +++ b/internal/scorecard/run_test.go @@ -153,15 +153,13 @@ func TestRunSequentialFail(t *testing.T) { func getFakeScorecard(parallel bool) Scorecard { return Scorecard{ - Config: v1alpha3.ScorecardConfiguration{ - Spec: v1alpha3.ScorecardConfigurationSpec{ - Stages: []v1alpha3.StageConfiguration{ - { - Parallel: parallel, - Tests: []v1alpha3.TestConfiguration{ - {}, - {}, - }, + Config: v1alpha3.Configuration{ + Stages: []v1alpha3.StageConfiguration{ + { + Parallel: parallel, + Tests: []v1alpha3.TestConfiguration{ + {}, + {}, }, }, }, diff --git a/internal/scorecard/scorecard.go b/internal/scorecard/scorecard.go index 2e0734663bd..03f731d433c 100644 --- a/internal/scorecard/scorecard.go +++ b/internal/scorecard/scorecard.go @@ -37,7 +37,7 @@ type TestRunner interface { } type Scorecard struct { - Config v1alpha3.ScorecardConfiguration + Config v1alpha3.Configuration Selector labels.Selector TestRunner TestRunner SkipCleanup bool @@ -67,7 +67,7 @@ func (o Scorecard) Run(ctx context.Context) (v1alpha3.TestList, error) { return testOutput, err } - for _, stage := range o.Config.Spec.Stages { + for _, stage := range o.Config.Stages { tests := o.selectTests(stage) if len(tests) == 0 { continue diff --git a/internal/scorecard/testdata/bundle.tar.gz b/internal/scorecard/testdata/bundle.tar.gz index 94425db64883e59772fbfa30d93cd3cddd6f146d..216613ef14b1a9ef343ee507d1448fb7a737ade1 100644 GIT binary patch literal 2768 zcmV;>3NQ5^iwFSqwHRLj1MM8ka^twsXMY9CIk+krQZHLdbW?2a#uYzQma{o*ZB0QW zBq0I`4oJ$Q-R!@o8{k`%CEH^;yURX|q$bd4^czihJnHiq3!|`g^ls1w7!1zwvoknv z7e7-t>huQfetU4%8}yIbo$h(}?1-GdA4@sJVMG-nM<4?wf8IWB*Yn>fc$xk^<~h_^ zFxQd+Q#6vwb_7U=sgS7^ci0NPap~J@>WLSvfeP;T&$j5l)9&>KW&L-1{ciV&wBKpZ zCHRBup9cH~qt_ux^&Z-eR7~QB*IIR`2o|(+yyU(+Bp1dcCdMR^smC6Rs7oU{JSK$t zzKm!D=n#JbLUnl(?vQIAmeF)@V?$Eo)T_4LwmZp)FrW?$3j};D zq;Q!!#_ef620U?mm(94(jYi2Y$bdHI|o6O|X>>g0)s{B^kI%?K?ANgU5|tCd`4+4-<5S z<452M5L^Mze(X(|B65~5OE^m0cNlC(!oWbtwVf%<OT?b#7?BI$F-j`=WZY(QBgV0GGtvdgq%zS) zx?3uDXtabXrnROm3B^)UApxGkdQs@<6h= z6Yqf$+yM&5$jX$4tbeBC9yu}b!ecI{(K=uWh8ur08%RA%i$hzA$NZ=u;%z>gZNEst zXgLtvq2QLC0ybm_5Mn?w+1oC)`V&FY^f&&S#=Qd!!Cy!zv*6kzdG%~^DpM)C0Ur>zW+?=LmG zhv5|0+{()R`08Qwc>nM$KRb;9fqQfgBOk8DH=u;i*SDkZHwo%NV@R!I37!JxI}%TH z9lp*45%R!V_5_3^Uyd9HL-F17B;l9G!!Xa9C?gdMRtk5Jg}+xa4oU`YrQxQ^QfW*n z{30S@uO8Wg98Lw~BUOIRd^eCB!c>_yW)3I>{b(gc>@i1B?o}qPSalY->x|Dm8dO}$LBpsp*_c9zMuP{hWHM<4=ylmkqlGdo9zfOP9Zu)g2+ z_MdM2JS0)}GHd*lI)Hn0X41pl9OL`2Hl+J`0A@Sp=Y{T`5FJMoi7M(kN|Xp zFH0B&jTO>LX{}Ra#*87lImgVBNpJ)nLbJGT*#JL~xM9DLPH55w$pMB?pUCNdm=oCq z`@TSggbNu97Ylm$8Nqx<&V49tQp&m;hD%~|pp+yrUqwtoC1j3zpMbmzaeWO$a19ds z1!{UoXp>v08e!s04#^^lpdx6s<~&Mm6p#8b7~;Bx4FSccF_w!h7mgsJ74o@7m9ya3 zA&M1ifdDPdxUUb>w&(sSQ+BaTFsTaM9?T{~Yg61RcQi=UFHNQy{&Vpn3Z?RJHF?Zx z6|I7@%(b-hX<;sS6cosTDWkX<9Naa<^q8i9ID(tfAP2B$k4LEKFENID(2q?nbWedy zSyHCBHo5iz>oIWwRouIa9VHxEDAL`6uj?t9fks++K~?D!)qCLtMwNUc?eXe!7FvC^ z-@MHq^_pO;)ecG7GOq6?R}YV5ba{FGczu62x*?PC)dl(P`s(I#V!Es4<&$@B8~ulC z4{IvwM?>-u`{X385UtQ%LV@bzFGfH^C+LETWhe*gLqcbDG(G9^+1eN<8Wk6ej9fQq z^B2M$i7|$fWjtlmnCtdwMCUafz0_K!QZB7k$Hq^)Fb&OT%GT0%KTq++Mc7LC1oFDr zSVW%Kpo$&zG<>4SH(SnmVSHMd4RIEV$00_aUbphh*b&C1IV+#(zQm}N%Bi&2wBKfg zCSB|{Lp`o!h;oWYg4bH2pPmAWu+tPDN00xS7-Q77h}={d*-%Dqqp!Bf4Vt-)zQrat z$Z8vXC!5^+O-R)sAuxJ1o>LqAijSyLbiLC^%WqR^t+axwdrg+$k`tJz>q>9YjD-}= zcc2`l+p-$-(ba7OyC6-g?|A*^HTQpdonCYQ=Ma$TzwmnL^(Xy$*m3^vw5#X;&iQ%o ztU3Q50xGw4@+&~;x0!;d|0LBC(xW|%eHRYs*;{UV)fr+whl96T|3%*-*pouqcG48` zCe#a9NM1JT`Kb}_Jsl4EOGs16?3!-teq@{l$d3uL3!N4}fjLbDdnpAQYAza!NE6zZ z1fHxbx(M<0ibIvVmohW~^HNUFjT;T~g^@Vi@xXSkNQW zRtc`OF*l|A{#;ApS*BpL^2XP8)?s*w@!IK!<=#om{s!(h=9?8hIs$(zRP+Bbjw11mtH77n|L5&?Q~w_X>h+J8 zD=d6z@l7n@<@)arx=sK0Ah1jS?KXbC?L4rf{%?1|*~|C8;A^j`{|^Dh`(LZ{mKXkw z*k&CMc3@#sv6yjb5DbYw=l<91N>}-Ba?-bkW(-bAY6XoLcI<)uw?B9a{!rM%|7jR9 z6}=$@?DBsI_tpEKys%u*SPyhfLYG>8} literal 2772 zcmV;_3M=&=iwFP}bQfO$1MM7LbK|!0p8YE@?t^FABPCjv)ap&Ooz&B_t+CwnA(Od* zNJv5r5CkY>m#eQP}pHBlmXHgt=KD-%p0fxgqemcW` zxBQvG2d6jeo}HaKeFw%lr+vryK>BY-QUOU6Q$@%J;DM4qZy&d-|91+0E`Oi-E_Ij8 zvt-B=jis_(0o-9KVrs=*wt{b52KJiz;#q5;fCq!KE%JA|z22}Y|I^-}JN!VpZ?xwM z{GZF8hWrO3*AdC&9@(x`OyY;%SvgdMOX?gw@W309bE6VdqY~-VBcH|8qcI&F5kiAN z#x#cJ2!BF@@GbV0`k`Rai2T>mjfDJ|eiHIa{*931f}rCOIo29!r?V^c9Dgj&LgH}z zb*9AC?us#kBIm`jgd@iTm%(-<3=D-_+qu9z9sz6DH=5KbXh_}@(s(%{QMu`> z&SetBC+WOQHl=MDHcq6Mqb_kKUfe=1lU5~%tM5qT%Ahb>-SPH1`SUl2o z^fDDhC>(3m5)jgOId)xi#Wzo*gkL_7q9SS{k5nR9#oa*^{$9x>tSGpZxtlskl`$3f ziw!Ax&5a$1;Z#68Qq_JIc%kIrrmDCxazG*1NhpSgl^zcz8j>a)OH=X~MQuug)j zSY;>#rde$_Fa-QerO5+7S8D=;sQeZ{WR-CAf|Zse^y-KoD}%NQfrgchY!O4r(^uOm z^Fj8k)||QYdOJpDHcv$Kd*l>sl!NB;U{N|+>}Fl6o9XNw+pce@#)jugQTx@}E}rrZ zyQSZ3VGTQ}@%dXifgOB%rK)e!-DVSbDJ|Rb{3i%I`SZ!s3%{SuJR@clIatBaBFDVO zq}L>-Q4)qy#c&2Au}aM9CMmsr4Dy6w$IvNQt6*D4&ndW~((vm<#C%`eVwn$8(;L+4f6k!aZ|i@DfO7p0zO%28)9IYle)wx-bJ9%B6izKEnk~tX&rS-0m1>?!p4?R=OD$e+?t$cxVg8-ll ze0jhqNvsrBDr=n}F=h= zSoZ}aBwETuc$m<`&lu*rauGmklVR4?FkBLg1C=0&@hWBtG9h!+`v~Yg@atBAZ-8)(8{lazvJK3>iVEv*2;2qj=O$KoQp+EC?t*OE6vRcyI&}oro_is@x^V z3Q?k12N-B+!UKJnwteqUxvu`OkBkC7Fl7Wchl9Jum>tvf4_9!L8WaE;?eiEV{Ut$n5Aw0exvnXY z8B5a?&n8y^wE9e(Ll*buY)1}97P53Vg;5pT$ZEX$oP}Cn zuQzYAN4+K}YrR5J)r_l~>E+!$8DCsn-Cy0_j6ad-w~hM4 zm5(_U4dM~GO9FD7Wr$YfJwSpQ;4el%LdR%=isel%aXZRrWjG@~J< z^Bo9B>9VYDdGGADgDVLXR}w_5*2 zzarQZLsoXu7>Xj)Ge}4;8}KQ|au6YU_Sz+$Hdj39}0w7C(YI%>;Wc z1{-oNnuu5<+7|_$tZTA}@Os6i%G*mA8iBc#)3akOgB(Yvjo*KGh~N0_FT(TR|2l)g zX&e6!0dA&|C{Ajy$$Xs~aqUlvJ0*i`P}v!FjW;_E?Ysx(^monRe7)D@F;A zffzkHZ`7-9U?}h(Y7bKQmPskCqBQpOMqo;98;wZbXD_buB5O%kuiZ`)p7-8Se%tiA z4drOwXFpocOOLc)vY9%LciM?A!oPYi8(7Rr@9p8AhI;&eNa9$0<1Fy``QO=a*yjHS zfoA#R;|dEeExw5+JYW8h2zFcf9|CsCzuU#n*PRD;eW1{T~AMvj5iN|FF<(|N38jeEAlYz@7GA|NkGHAwX;Y zhk&yEKO~;C;S^na^L-CnpW?~)s3_N+9mU*dGJ&(NbJ{&Z9>(YR2vmxRpaIMNKIFZ( a*Fp;|w9rBeEws==4gLoofR7&lPyhgeV`@hL diff --git a/internal/scorecard/testdata/bundle/tests/scorecard/config.yaml b/internal/scorecard/testdata/bundle/tests/scorecard/config.yaml index 7107497532a..0a8ac76302f 100644 --- a/internal/scorecard/testdata/bundle/tests/scorecard/config.yaml +++ b/internal/scorecard/testdata/bundle/tests/scorecard/config.yaml @@ -2,49 +2,48 @@ kind: ScorecardConfiguration apiversion: scorecard.operatorframework.io/v1alpha3 metadata: name: config -spec: - stages: - - parallel: true - tests: - - image: quay.io/operator-framework/scorecard-test:dev - entrypoint: - - scorecard-test - - basic-check-spec - labels: - suite: basic - test: basic-check-spec-test - - image: quay.io/operator-framework/scorecard-test:dev - entrypoint: - - scorecard-test - - olm-bundle-validation - labels: - suite: olm - test: olm-bundle-validation-test - - image: quay.io/operator-framework/scorecard-test:dev - entrypoint: - - scorecard-test - - olm-crds-have-validation - labels: - suite: olm - test: olm-crds-have-validation-test - - image: quay.io/operator-framework/scorecard-test:dev - entrypoint: - - scorecard-test - - olm-crds-have-resources - labels: - suite: olm - test: olm-crds-have-resources-test - - image: quay.io/operator-framework/scorecard-test:dev - entrypoint: - - scorecard-test - - olm-spec-descriptors - labels: - suite: olm - test: olm-spec-descriptors-test - - image: quay.io/operator-framework/scorecard-test:dev - entrypoint: - - scorecard-test - - olm-status-descriptors - labels: - suite: olm - test: olm-status-descriptors-test +stages: +- parallel: true + tests: + - image: quay.io/operator-framework/scorecard-test:dev + entrypoint: + - scorecard-test + - basic-check-spec + labels: + suite: basic + test: basic-check-spec-test + - image: quay.io/operator-framework/scorecard-test:dev + entrypoint: + - scorecard-test + - olm-bundle-validation + labels: + suite: olm + test: olm-bundle-validation-test + - image: quay.io/operator-framework/scorecard-test:dev + entrypoint: + - scorecard-test + - olm-crds-have-validation + labels: + suite: olm + test: olm-crds-have-validation-test + - image: quay.io/operator-framework/scorecard-test:dev + entrypoint: + - scorecard-test + - olm-crds-have-resources + labels: + suite: olm + test: olm-crds-have-resources-test + - image: quay.io/operator-framework/scorecard-test:dev + entrypoint: + - scorecard-test + - olm-spec-descriptors + labels: + suite: olm + test: olm-spec-descriptors-test + - image: quay.io/operator-framework/scorecard-test:dev + entrypoint: + - scorecard-test + - olm-status-descriptors + labels: + suite: olm + test: olm-status-descriptors-test diff --git a/pkg/apis/scorecard/v1alpha3/configuration_types.go b/pkg/apis/scorecard/v1alpha3/configuration_types.go index c87cc85c0a6..0228eb8d1a1 100644 --- a/pkg/apis/scorecard/v1alpha3/configuration_types.go +++ b/pkg/apis/scorecard/v1alpha3/configuration_types.go @@ -18,35 +18,37 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -// scorecardConfigKind is the default scorecard componentconfig kind. -const ScorecardConfigurationKind = "ScorecardConfiguration" +// ConfigurationKind is the default scorecard componentconfig kind. +const ConfigurationKind = "ScorecardConfiguration" -// ScorecardConfiguration is the Schema for the scorecardconfigurations API -type ScorecardConfiguration struct { +// Configuration represents the set of test configurations which scorecard would run. +type Configuration struct { metav1.TypeMeta `json:",inline" yaml:",inline"` - // Do not use metav1.ObjectMeta because this object should not be apply-able. + + // Do not use metav1.ObjectMeta because this "object" should not be treated as an actual object. Metadata struct { Name string `json:"name,omitempty" yaml:"name,omitempty"` } `json:"metadata,omitempty" yaml:"metadata,omitempty"` - Spec ScorecardConfigurationSpec `json:"spec,omitempty" yaml:"spec,omitempty"` -} - -// ScorecardConfigurationSpec represents the set of test configurations which scorecard would run based on user input. -type ScorecardConfigurationSpec struct { + // Stages is a set of test stages to run. Once a stage is finished, the next stage in the slice will be run. Stages []StageConfiguration `json:"stages" yaml:"stages"` } +// StageConfiguration configures a set of tests to be run. type StageConfiguration struct { - Parallel bool `json:"parallel,omitempty" yaml:"parallel,omitempty"` - Tests []TestConfiguration `json:"tests" yaml:"tests"` + // Parallel, if true, will run each test in tests in parallel. + // The default is to wait until a test finishes to run the next. + Parallel bool `json:"parallel,omitempty" yaml:"parallel,omitempty"` + // Tests are a list of tests to run. + Tests []TestConfiguration `json:"tests" yaml:"tests"` } +// TestConfiguration configures a specific scorecard test, identified by entrypoint. type TestConfiguration struct { - // Image is the name of the testimage + // Image is the name of the test image. Image string `json:"image" yaml:"image"` - // Entrypoint is list of commands and arguments passed to the test image + // Entrypoint is a list of commands and arguments passed to the test image. Entrypoint []string `json:"entrypoint,omitempty" yaml:"entrypoint,omitempty"` - // Labels that further describe the test and enable selection + // Labels further describe the test and enable selection. Labels map[string]string `json:"labels,omitempty" yaml:"labels,omitempty"` } From 76c9817acff9f9cf7ea2095c9f10d3cfe107437c Mon Sep 17 00:00:00 2001 From: Eric Stroczynski Date: Wed, 22 Jul 2020 15:26:51 -0700 Subject: [PATCH 5/7] update scorecard docs --- internal/plugins/scorecard/init.go | 2 +- .../advanced-topics/scorecard/custom-tests.md | 129 ++++++++++++------ .../advanced-topics/scorecard/scorecard.md | 37 +++-- 3 files changed, 108 insertions(+), 60 deletions(-) diff --git a/internal/plugins/scorecard/init.go b/internal/plugins/scorecard/init.go index 1588d95d7a5..216719161b1 100644 --- a/internal/plugins/scorecard/init.go +++ b/internal/plugins/scorecard/init.go @@ -165,7 +165,7 @@ func generate(operatorName, testImageTag, outputDir string) error { type jsonPatches []jsonPatchObject // jsonPatchObject is a JSON 6902 patch object specific to the scorecard's test configuration. -// See https://tools.ietf.org/html/rfc6902 for details. +// https://kubernetes-sigs.github.io/kustomize/api-reference/kustomization/patchesjson6902/ for details. type jsonPatchObject struct { Op string `json:"op"` Path string `json:"path"` diff --git a/website/content/en/docs/advanced-topics/scorecard/custom-tests.md b/website/content/en/docs/advanced-topics/scorecard/custom-tests.md index 2ded3087249..13ccc3d7d44 100644 --- a/website/content/en/docs/advanced-topics/scorecard/custom-tests.md +++ b/website/content/en/docs/advanced-topics/scorecard/custom-tests.md @@ -3,7 +3,7 @@ title: Writing Custom Scorecard Tests weight: 50 --- -This guide outlines the steps which can be followed to extend the existing scorecard tests and implement operator specific custom tests. +This guide outlines the steps which can be followed to extend the existing scorecard tests and implement operator specific custom tests. ## Run scorecard with custom tests: @@ -14,6 +14,18 @@ The following steps explain creating of a custom test image which can be used wi The sample test image repository present [here][custom_scorecard_repo] has the following project structure: ``` +$ tree . +. +... +├── config +| ... +│   └── scorecard +│      ├── bases +│      │   └── config.yaml +│      ├── kustomization.yaml +│      └── patches +│      ├── basic.config.yaml +│      └── olm.config.yaml ├── Makefile ├── bundle │   ├── manifests @@ -41,10 +53,11 @@ The sample test image repository present [here][custom_scorecard_repo] has the f └── tests.go ``` +1. `config/scorecard` - Contains a kustomization for generating a config from a base and set of overlays. 1. `bundle/` - Contains bundle manifests and metadata under test. -2. `bundle/tests/scorecard/config.yaml` - Configuration yaml to define and run scorecard tests. -3. `images/custom-scorecard-tests/cmd/test/main.go` - Scorecard test binary. -4. `internal/tests/tests.go` - Contains the implementation of custom tests specific to the operator. +1. `bundle/tests/scorecard/config.yaml` - Configuration file generated by `make bundle` from the `config/scorecard` kustomization. +1. `images/custom-scorecard-tests/cmd/test/main.go` - Scorecard test binary. +1. `internal/tests/tests.go` - Contains the implementation of custom tests specific to the operator. #### Writing custom test logic: @@ -56,48 +69,73 @@ The `tests.go` file is where the custom tests are implemented in the sample test package tests import ( - "github.com/operator-framework/operator-registry/pkg/registry" - scapiv1alpha3 "github.com/operator-framework/operator-sdk/pkg/apis/scorecard/v1alpha3" + "github.com/operator-framework/operator-registry/pkg/registry" + scapiv1alpha3 "github.com/operator-framework/operator-sdk/pkg/apis/scorecard/v1alpha3" ) const ( - CustomTest1Name = "customtest1" + CustomTest1Name = "customtest1" ) -// CustomTest1 +// CustomTest1 func CustomTest1(bundle registry.Bundle) scapiv1alpha3.TestStatus { - r := scapiv1alpha3.TestResult{} - r.Name = CustomTest1Name - r.Description = "Custom Test 1" - r.State = scapiv1alpha3.PassState - r.Errors = make([]string, 0) - r.Suggestions = make([]string, 0) + r := scapiv1alpha3.TestResult{} + r.Name = CustomTest1Name + r.Description = "Custom Test 1" + r.State = scapiv1alpha3.PassState + r.Errors = make([]string, 0) + r.Suggestions = make([]string, 0) // Implement relevant custom test logic here - return wrapResult(r) + return wrapResult(r) } ``` ### Scorecard Configuration file: -The [configuration file][config_yaml] includes test definitions and metadata to run the test. For the example `CustomTest1` function, the following fields should be specified in `config.yaml`. +The [configuration file][config_yaml] includes test definitions and metadata to run the test. +This file is constructed using a kustomization under `config/scorecard`, with overlays for test sets. + +For the example `CustomTest1` function, add the following to `config/scorecard/customtest1.config.yaml`. ```yaml -stages: -- tests: - - image: quay.io/username/custom-scorecard-tests:dev +- op: add + path: /stages/0/tests/0 + value: + image: quay.io//custom-scorecard-tests:latest entrypoint: - custom-scorecard-tests - customtest1 labels: suite: custom test: customtest1 - ``` +``` The important fields to note here are: 1. `image` - name and tag of the test image which was specified in the Makefile. -2. `labels` - the name of the `test` and `suite` the test function belongs to. This can be specified in the `operator-sdk scorecard` command to run the desired test. +2. `labels` - the name of the `test` and `suite` the test function belongs to. +This can be specified in the `operator-sdk scorecard` command to run the desired test. + +Next, add a [JSON 6902 patch][kustomize-patchJson6902] to your `config/scorecard/kustomization.yaml`: + +```yaml +patchesJson6902: +... +- path: patches/customtest1.config.yaml + target: + group: scorecard.operatorframework.io + version: v1alpha3 + kind: ScorecardConfiguration + name: config +``` + +Once you run `make bundle`, the `bundle/tests/scorecard/config.yaml` will be (re)generated with your custom test. + +If generating a config file outside of the on-disk bundle, you can run: +```console +$ kustomize build config/scorecard > path/to/config.yaml +``` **Note**: The default location of `config.yaml` inside the bundle is `/tests/scorecard/config.yaml`. It can be overridden using the `--config` flag. For more details regarding the configuration file refer to [user docs][user_doc]. @@ -106,10 +144,10 @@ The important fields to note here are: The scorecard test image implementation requires the bundle under test to be present in the test image. The `apimanifests.GetBundleFromDir()` function reads the pod's bundle to fetch the manifests and scorecard configuration from desired path. ```Go - cfg, err := apimanifests.GetBundleFromDir(scorecard.PodBundleRoot) - if err != nil { - log.Fatal(err.Error()) - } +cfg, err := apimanifests.GetBundleFromDir(scorecard.PodBundleRoot) +if err != nil { + log.Fatal(err.Error()) +} ``` The scorecard binary uses `config.yaml` file to locate tests and execute the them as Pods which scorecard creates. Custom test images are included into Pods that scorecard creates, passing in the bundle contents on a shared mount point to the test image container. The specific custom test that is executed is driven by the config.yaml's entry-point command and arguments. @@ -118,11 +156,11 @@ An example scorecard binary is present [here][scorecard_binary]. The names with which the tests are identified in `config.yaml` and would be passed in the `scorecard` command, are to be specified here. ```Go -... +... switch entrypoint[0] { case tests.CustomTest1Name: - result = tests.CustomTest1(cfg) - ... + result = tests.CustomTest1(cfg) + ... } ... ``` @@ -131,9 +169,9 @@ The result of the custom tests which is in `scapiv1alpha3.TestResult` format, is ```Go prettyJSON, err := json.MarshalIndent(result, "", " ") - if err != nil { - log.Fatal("Failed to generate json", err) - } +if err != nil { + log.Fatal("Failed to generate json", err) +} fmt.Printf("%s\n", string(prettyJSON)) ``` @@ -141,11 +179,11 @@ The names of the custom tests are also included in `printValidTests()` function: ```Go func printValidTests() (result scapiv1alpha3.TestStatus) { -... - str := fmt.Sprintf("Valid tests for this image include: %s, - tests.CustomTest1Name - result.Errors = append(result.Errors, str") -... + ... + str := fmt.Sprintf("Valid tests for this image include: %s", tests.CustomTest1Name) + result.Errors = append(result.Errors, str) + ... +} ``` @@ -166,7 +204,7 @@ docker push //:tag The `operator-sdk scorecard` command is used to execute the scorecard tests by specifying the location of test bundle in the command. The name or suite of the tests which are to be executed can be specified with the `--selector` flag. The command will create scorecard pods with the image specified in `config.yaml` for the respective test. For example, the `CustomTest1Name` test provides the following json output. ```console -operator-sdk scorecard --selector=suite=custom -o json --wait-time=32s --skip-cleanup=false +$ operator-sdk scorecard --selector=suite=custom -o json --wait-time=32s --skip-cleanup=false { "metadata": { "creationTimestamp": null @@ -187,7 +225,7 @@ operator-sdk scorecard --selector=suite=custom -o json --w ### Debugging scorecard custom tests -The `--skip-cleanup` flag can be used when executing the `operator-sdk scorecard` command to cause the scorecard created test pods to be unremoved. +The `--skip-cleanup` flag can be used when executing the `operator-sdk scorecard` command to cause the scorecard created test pods to be unremoved. This is useful when debugging or writing new tests so that you can view the test logs or the pod manifests. @@ -208,14 +246,14 @@ namespaces for your test but instead considers these resources to be outside its scope. You can however implement whatever service accounts your tests require and then specify that service account from the command line using the service-account flag: -``` -operator-sdk scorecard --service-account=mycustomsa +```console +$ operator-sdk scorecard --service-account=mycustomsa ``` Also, you can set up a non-default namespace that your tests will be executed within using the following namespace flag: -``` -operator-sdk scorecard --namespace=mycustomns +```console +$ operator-sdk scorecard --namespace=mycustomns ``` If you do not specify either of these flags, the default namespace @@ -231,9 +269,9 @@ https://github.com/operator-framework/operator-sdk/blob/master/pkg/apis/scorecar ### Accessing the Kube API Within your custom tests you might require connecting to the Kube API. -In golang, you could use the [client-go][client_go] API for example to -check Kube resources within your tests, or even create custom resources. Your -custom test image is being executed within a Pod, so you can use an in-cluster +In golang, you could use the [client-go][client_go] API for example to +check Kube resources within your tests, or even create custom resources. Your +custom test image is being executed within a Pod, so you can use an in-cluster connection to invoke the Kube API. @@ -246,3 +284,4 @@ connection to invoke the Kube API. [user_doc]: /docs/advanced-topics/scorecard/scorecard/ [scorecard_binary]: https://github.com/operator-framework/operator-sdk/blob/master/internal/scorecard/examples/custom-scorecard-tests/images/custom-scorecard-tests/cmd/test/main.go [sample_makefile]: https://github.com/operator-framework/operator-sdk/blob/master/internal/scorecard/examples/custom-scorecard-tests/Makefile +[kustomize-patchJson6902]: https://kubernetes-sigs.github.io/kustomize/api-reference/kustomization/patchesjson6902/ diff --git a/website/content/en/docs/advanced-topics/scorecard/scorecard.md b/website/content/en/docs/advanced-topics/scorecard/scorecard.md index 1005c633717..7fa6b14f1e4 100644 --- a/website/content/en/docs/advanced-topics/scorecard/scorecard.md +++ b/website/content/en/docs/advanced-topics/scorecard/scorecard.md @@ -30,12 +30,20 @@ require if the tests are designed for resource creation. ## Running the Scorecard -1. Define a scorecard configuration file by running +1. A default set of kustomize files should have been scaffolded by `operator-sdk init`. +If that is not the case, run `operator-sdk init` as you would have to initialize your project +and copy scaffolded files: ```sh - $ operator-sdk generate kustomize scorecard + $ TMP_PROJECT="$(mktemp -d)/" + $ mkdir "$TMP_PROJECT" + $ pushd "$TMP_PROJECT" + $ operator-sdk init + $ popd + $ cp -r "$TMP_PROJECT"/config/scorecard ./config/ ``` - See the [config file section](#config-file) for an explanation of the configuration file format. -1. Generate [bundle][quickstart-bundle] manifests and metadata for your Operator. +The default config generated by this kustomization can be immediately run against your operator. +See the [config file section](#config-file) for an explanation of the configuration file format. +1. (Re)enerate [bundle][quickstart-bundle] manifests and metadata for your Operator. `make bundle` will automatically add scorecard annotations to your bundle's metadata, which is used by the `scorecard` command to run tests. 1. Execute the [`scorecard` command][cli-scorecard]. See the [command args section](#command-args) @@ -43,7 +51,7 @@ for an overview of command invocation. ## Configuration -The scorecard test execution is driven by a configuration file named `config.yaml`. +The scorecard test execution is driven by a configuration file named `config.yaml`, generated by `make bundle`. The configuration file is located at the following location within your bundle directory (`bundle/` by default): ```sh $ tree ./bundle @@ -56,23 +64,24 @@ $ tree ./bundle ### Config File -A complete scorecard configuration file can be found [here][sample-config] -and used for running the scorecard pre-defined tests that ship with the SDK. - -A sample of the scorecard configuration file may look as follows: +The following YAML spec is an example of the scorecard configuration file: ```yaml +kind: ScorecardConfiguration +apiversion: scorecard.operatorframework.io/v1alpha3 +metadata: + name: config stages: - parallel: true tests: - - image: quay.io/operator-framework/scorecard-test:dev + - image: quay.io/operator-framework/scorecard-test:latest entrypoint: - scorecard-test - basic-check-spec labels: suite: basic test: basic-check-spec-test - - image: quay.io/operator-framework/scorecard-test:dev + - image: quay.io/operator-framework/scorecard-test:latest entrypoint: - scorecard-test - olm-bundle-validation @@ -186,7 +195,7 @@ See an example of the JSON format produced by a scorecard test: "kind": "Test", "apiVersion": "scorecard.operatorframework.io/v1alpha3", "spec": { - "image": "quay.io/operator-framework/scorecard-test:dev", + "image": "quay.io/operator-framework/scorecard-test:latest", "entrypoint": [ "scorecard-test", "olm-bundle-validation" @@ -216,7 +225,7 @@ See an example of the text format produced by a scorecard test: ``` -------------------------------------------------------------------------------- -Image: quay.io/operator-framework/scorecard-test:dev +Image: quay.io/operator-framework/scorecard-test:latest Entrypoint: [scorecard-test olm-bundle-validation] Labels: "suite":"olm" @@ -250,7 +259,7 @@ Scorecard will execute custom tests if they follow these mandated conventions: * tests can obtain the bundle contents at a shared mount point of /bundle * tests can access the Kubernetes API using an in-cluster client connection -See [here][custom-image] for an example of a custom test image written in golang. +See [here][custom-image] for an example of a custom test image written in Go. Writing custom tests in other programming languages is possible if the test image follows the above guidelines. From 3d7fdb526a56af231cfd9c458df8eacd6bbe8d71 Mon Sep 17 00:00:00 2001 From: Eric Stroczynski Date: Wed, 22 Jul 2020 15:46:51 -0700 Subject: [PATCH 6/7] update test bundle --- internal/scorecard/testdata/bundle.tar.gz | Bin 2768 -> 2768 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/internal/scorecard/testdata/bundle.tar.gz b/internal/scorecard/testdata/bundle.tar.gz index 216613ef14b1a9ef343ee507d1448fb7a737ade1..081bf80cd41ed92615bfe81c6298b467f5221a06 100644 GIT binary patch delta 15 Wcmca0dO?&;zMF&L%E66nr?>zty#=EH delta 15 Wcmca0dO?&;zMF&L>(-5Ir?>zuM+LY5 From 75b2dfc6874ac2b141f0486c3e207a4954838599 Mon Sep 17 00:00:00 2001 From: Eric Stroczynski Date: Thu, 23 Jul 2020 12:21:10 -0700 Subject: [PATCH 7/7] updates based on PR comments --- cmd/operator-sdk/generate/bundle/bundle.go | 2 +- internal/generate/collector/collect.go | 10 +- internal/plugins/golang/v2/init.go | 11 +- internal/plugins/helm/v1/init.go | 8 +- internal/plugins/scorecard/init.go | 88 +++++++------- .../bundle/tests/scorecard/config.yaml | 2 +- internal/scorecard/formatting.go | 4 +- internal/scorecard/scorecard.go | 4 +- internal/scorecard/testdata/bundle.tar.gz | Bin 2768 -> 2762 bytes .../bundle/tests/scorecard/config.yaml | 2 +- .../scorecard/v1alpha3/configuration_types.go | 3 +- pkg/apis/scorecard/v1alpha3/test_types.go | 14 +-- .../v1alpha3/zz_generated.deepcopy.go | 107 +++++++++++++----- .../advanced-topics/scorecard/custom-tests.md | 38 +++++-- .../advanced-topics/scorecard/scorecard.md | 2 +- 15 files changed, 174 insertions(+), 121 deletions(-) diff --git a/cmd/operator-sdk/generate/bundle/bundle.go b/cmd/operator-sdk/generate/bundle/bundle.go index c9cb0117f13..10316ac9372 100644 --- a/cmd/operator-sdk/generate/bundle/bundle.go +++ b/cmd/operator-sdk/generate/bundle/bundle.go @@ -307,7 +307,7 @@ func (c bundleCmd) generateMetadata(cfg *config.Config, manifestsDir, outputDir return nil } -// NB(estroz): these updates need to be atomic because the bundle's Dockerfile and annotations.yaml +// TODO(estroz): these updates need to be atomic because the bundle's Dockerfile and annotations.yaml // cannot be out-of-sync. func updateMetadata(cfg *config.Config, bundleRoot string) error { bundleLabels := metricsannotations.MakeBundleMetadataLabels(cfg) diff --git a/internal/generate/collector/collect.go b/internal/generate/collector/collect.go index 35f2c5f8381..77f26674f64 100644 --- a/internal/generate/collector/collect.go +++ b/internal/generate/collector/collect.go @@ -90,7 +90,9 @@ func (c *Manifests) UpdateFromDirs(deployDir, crdsDir string) error { case "MutatingWebhookConfiguration": err = c.addMutatingWebhookConfigurations(manifest) case scorecardv1alpha3.ConfigurationKind: - err = c.addScorecardConfig(manifest) + if gvk.GroupVersion() == scorecardv1alpha3.SchemeGroupVersion { + err = c.addScorecardConfig(manifest) + } default: err = c.addOthers(manifest) } @@ -152,7 +154,9 @@ func (c *Manifests) UpdateFromReader(r io.Reader) error { case "MutatingWebhookConfiguration": err = c.addMutatingWebhookConfigurations(manifest) case scorecardv1alpha3.ConfigurationKind: - err = c.addScorecardConfig(manifest) + if gvk.GroupVersion() == scorecardv1alpha3.SchemeGroupVersion { + err = c.addScorecardConfig(manifest) + } default: err = c.addOthers(manifest) } @@ -273,7 +277,7 @@ func (c *Manifests) addScorecardConfig(rawManifest []byte) error { return err } if c.ScorecardConfig.Metadata.Name != "" { - return errors.New("duplicate ScorecardConfigurations in collector input") + return errors.New("duplicate scorecard configurations in collector input") } c.ScorecardConfig = cfg return nil diff --git a/internal/plugins/golang/v2/init.go b/internal/plugins/golang/v2/init.go index 4cc5c3511dc..466d27ba662 100644 --- a/internal/plugins/golang/v2/init.go +++ b/internal/plugins/golang/v2/init.go @@ -16,9 +16,6 @@ package v2 import ( "fmt" - "os" - "path/filepath" - "strings" "github.com/spf13/pflag" "sigs.k8s.io/kubebuilder/pkg/model/config" @@ -54,14 +51,8 @@ func (p *initPlugin) Run() error { return err } - // Assume projectName was validated by go.kubebuilder.io. - wd, err := os.Getwd() - if err != nil { - return fmt.Errorf("error getting the current path: %v", err) - } - projectName := strings.ToLower(filepath.Base(wd)) // Run the scorecard "phase 2" plugin. - if err := scorecard.RunInit(projectName); err != nil { + if err := scorecard.RunInit(p.config); err != nil { return err } diff --git a/internal/plugins/helm/v1/init.go b/internal/plugins/helm/v1/init.go index 352032a6cf1..f60ea767079 100644 --- a/internal/plugins/helm/v1/init.go +++ b/internal/plugins/helm/v1/init.go @@ -133,14 +133,8 @@ func (p *initPlugin) Run() error { return err } - // Assume projectName was validated by Validate(). - wd, err := os.Getwd() - if err != nil { - return fmt.Errorf("error getting the current path: %v", err) - } - projectName := strings.ToLower(filepath.Base(wd)) // Run the scorecard "phase 2" plugin. - if err := scorecard.RunInit(projectName); err != nil { + if err := scorecard.RunInit(p.config); err != nil { return err } diff --git a/internal/plugins/scorecard/init.go b/internal/plugins/scorecard/init.go index 216719161b1..b2657edd35a 100644 --- a/internal/plugins/scorecard/init.go +++ b/internal/plugins/scorecard/init.go @@ -22,7 +22,8 @@ import ( "path/filepath" "text/template" - "github.com/operator-framework/operator-registry/pkg/registry" + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/kubebuilder/pkg/model/config" "sigs.k8s.io/yaml" "github.com/operator-framework/operator-sdk/internal/scaffold/kustomize" @@ -30,9 +31,10 @@ import ( "github.com/operator-framework/operator-sdk/pkg/apis/scorecard/v1alpha3" ) -// kustomization.yaml file template for the scorecard componentconfig. This should always be written to -// config/scorecard/kustomization.yaml since it only references files in config. -const scorecardKustomizationTemplate = `resources: +const ( + // kustomization.yaml file template for the scorecard componentconfig. This should always be written to + // config/scorecard/kustomization.yaml since it only references files in config. + scorecardKustomizationTemplate = `resources: {{- range $i, $path := .ResourcePaths }} - {{ $path }} {{- end }} @@ -47,24 +49,28 @@ patchesJson6902: {{- end }} ` + // YAML file fragment to append to kustomization.yaml files. + kubebuilderScaffoldMarkerFragment = "# +kubebuilder:scaffold:patchesJson6902\n" +) + const ( // defaultTestImageTag points to the latest-released image. // TODO: change the tag to "latest" once config scaffolding is in a release, // as the new config spec won't work with the current latest image. defaultTestImageTag = "quay.io/operator-framework/scorecard-test:master" - // scorecardConfigName is the default scorecard componentconfig's metadata.name, + // defaultConfigName is the default scorecard componentconfig's metadata.name, // which must be set on all kustomize-able bases. This name is only used for // `kustomize build` pattern match and not for on-cluster creation. - scorecardConfigName = "config" + defaultConfigName = "config" ) // defaultDir is the default directory in which to generate kustomize bases and the kustomization.yaml. var defaultDir = filepath.Join("config", "scorecard") // RunInit scaffolds kustomize files for kustomizing a scorecard componentconfig. -func RunInit(projectName string) error { - return generate(projectName, defaultTestImageTag, defaultDir) +func RunInit(*config.Config) error { + return generate(defaultTestImageTag, defaultDir) } // scorecardKustomizationValues holds data required to generate a scorecard's kustomization.yaml. @@ -76,12 +82,18 @@ type scorecardKustomizationValues struct { // kustomizationJSON6902Patch holds path and target data to write a patchesJson6902 list in a kustomization.yaml. type kustomizationJSON6902Patch struct { Path string - Target registry.DefinitionKey + Target patchTarget +} + +// patchTarget holds target data for a kustomize patch. +type patchTarget struct { + schema.GroupVersionKind + Name string } // generate scaffolds kustomize bundle bases and a kustomization.yaml. // TODO(estroz): refactor this to be testable (in-mem fs) and easier to read. -func generate(operatorName, testImageTag, outputDir string) error { +func generate(testImageTag, outputDir string) error { kustomizationValues := scorecardKustomizationValues{} @@ -91,7 +103,7 @@ func generate(operatorName, testImageTag, outputDir string) error { return err } - configBase := newScorecardConfigurationBase() + configBase := newConfigurationBase(defaultConfigName) b, err := yaml.Marshal(configBase) if err != nil { return fmt.Errorf("error marshaling default config: %v", err) @@ -102,11 +114,9 @@ func generate(operatorName, testImageTag, outputDir string) error { return fmt.Errorf("error writing default scorecard config: %v", err) } kustomizationValues.ResourcePaths = append(kustomizationValues.ResourcePaths, relBasePath) - scorecardConfigTarget := registry.DefinitionKey{ - Group: v1alpha3.SchemeGroupVersion.Group, - Version: v1alpha3.SchemeGroupVersion.Version, - Kind: v1alpha3.ConfigurationKind, - Name: scorecardConfigName, + scorecardConfigTarget := patchTarget{ + GroupVersionKind: v1alpha3.SchemeGroupVersion.WithKind(v1alpha3.ConfigurationKind), + Name: defaultConfigName, } // Config patches. @@ -116,7 +126,7 @@ func generate(operatorName, testImageTag, outputDir string) error { } // Basic scorecard tests patch. - basicPatch := newBasicScorecardConfigurationPatch(operatorName, testImageTag) + basicPatch := newBasicConfigurationPatch(testImageTag) b, err = yaml.Marshal(basicPatch) if err != nil { return fmt.Errorf("error marshaling basic patch config: %v", err) @@ -131,7 +141,7 @@ func generate(operatorName, testImageTag, outputDir string) error { }) // OLM scorecard tests patch. - olmPatch := newOLMScorecardConfigurationPatch(operatorName, testImageTag) + olmPatch := newOLMConfigurationPatch(testImageTag) b, err = yaml.Marshal(olmPatch) if err != nil { return fmt.Errorf("error marshaling OLM patch config: %v", err) @@ -154,6 +164,8 @@ func generate(operatorName, testImageTag, outputDir string) error { if err = t.Execute(&buf, kustomizationValues); err != nil { return fmt.Errorf("error executing on default kustomize template: %v", err) } + // Append the kubebuilder scaffold marker to make updates to this file in the future. + buf.Write([]byte(kubebuilderScaffoldMarkerFragment)) if err := kustomize.Write(outputDir, buf.String()); err != nil { return fmt.Errorf("error writing default scorecard kustomization.yaml: %v", err) } @@ -172,11 +184,11 @@ type jsonPatchObject struct { Value v1alpha3.TestConfiguration `json:"value"` } -// newScorecardConfigurationBase returns a scorecard componentconfig object with one parallel stage. +// newConfigurationBase returns a scorecard componentconfig object with one parallel stage. // The returned object is intended to be marshaled and written to disk as a kustomize base. -func newScorecardConfigurationBase() (cfg v1alpha3.Configuration) { +func newConfigurationBase(configName string) (cfg v1alpha3.Configuration) { cfg.SetGroupVersionKind(v1alpha3.SchemeGroupVersion.WithKind(v1alpha3.ConfigurationKind)) - cfg.Metadata.Name = scorecardConfigName + cfg.Metadata.Name = configName cfg.Stages = []v1alpha3.StageConfiguration{ { Parallel: true, @@ -186,18 +198,16 @@ func newScorecardConfigurationBase() (cfg v1alpha3.Configuration) { return cfg } -func makeTestStageJSONPath(stageIdx, testIdx int) string { - return fmt.Sprintf("/stages/%d/tests/%d", stageIdx, testIdx) -} +const defaultJSONPath = "/stages/0/tests/-" -// newBasicScorecardConfigurationPatch returns default "basic" test configurations as JSON patch objects +// newBasicConfigurationPatch returns default "basic" test configurations as JSON patch objects // to be inserted into the componentconfig base as a first stage test element. // The returned patches are intended to be marshaled and written to disk as in a kustomize patch file. -func newBasicScorecardConfigurationPatch(operatorName, testImageTag string) (ps jsonPatches) { - for i, cfg := range makeDefaultBasicTestConfigs(operatorName, testImageTag) { +func newBasicConfigurationPatch(testImageTag string) (ps jsonPatches) { + for _, cfg := range makeDefaultBasicTestConfigs(testImageTag) { ps = append(ps, jsonPatchObject{ Op: "add", - Path: makeTestStageJSONPath(0, i), + Path: defaultJSONPath, Value: cfg, }) } @@ -205,15 +215,14 @@ func newBasicScorecardConfigurationPatch(operatorName, testImageTag string) (ps } // makeDefaultBasicTestConfigs returns all default "basic" test configurations. -func makeDefaultBasicTestConfigs(operatorName, testImageTag string) (cfgs []v1alpha3.TestConfiguration) { +func makeDefaultBasicTestConfigs(testImageTag string) (cfgs []v1alpha3.TestConfiguration) { for _, testName := range []string{"basic-check-spec"} { cfgs = append(cfgs, v1alpha3.TestConfiguration{ Image: testImageTag, Entrypoint: []string{"scorecard-test", testName}, Labels: map[string]string{ - "operator": operatorName, - "suite": "basic", - "test": fmt.Sprintf("%s-test", testName), + "suite": "basic", + "test": fmt.Sprintf("%s-test", testName), }, }) } @@ -221,14 +230,14 @@ func makeDefaultBasicTestConfigs(operatorName, testImageTag string) (cfgs []v1al return cfgs } -// newOLMScorecardConfigurationPatch returns default "olm" test configurations as JSON patch objects +// newOLMConfigurationPatch returns default "olm" test configurations as JSON patch objects // to be inserted into the componentconfig base as a first stage test element. // The returned patches are intended to be marshaled and written to disk as in a kustomize patch file. -func newOLMScorecardConfigurationPatch(operatorName, testImageTag string) (ps jsonPatches) { - for i, cfg := range makeDefaultOLMTestConfigs(operatorName, testImageTag) { +func newOLMConfigurationPatch(testImageTag string) (ps jsonPatches) { + for _, cfg := range makeDefaultOLMTestConfigs(testImageTag) { ps = append(ps, jsonPatchObject{ Op: "add", - Path: makeTestStageJSONPath(0, i), + Path: defaultJSONPath, Value: cfg, }) } @@ -236,7 +245,7 @@ func newOLMScorecardConfigurationPatch(operatorName, testImageTag string) (ps js } // makeDefaultOLMTestConfigs returns all default "olm" test configurations. -func makeDefaultOLMTestConfigs(operatorName, testImageTag string) (cfgs []v1alpha3.TestConfiguration) { +func makeDefaultOLMTestConfigs(testImageTag string) (cfgs []v1alpha3.TestConfiguration) { for _, testName := range []string{ "olm-bundle-validation", "olm-crds-have-validation", @@ -248,9 +257,8 @@ func makeDefaultOLMTestConfigs(operatorName, testImageTag string) (cfgs []v1alph Image: testImageTag, Entrypoint: []string{"scorecard-test", testName}, Labels: map[string]string{ - "operator": operatorName, - "suite": "olm", - "test": fmt.Sprintf("%s-test", testName), + "suite": "olm", + "test": fmt.Sprintf("%s-test", testName), }, }) } diff --git a/internal/scorecard/examples/custom-scorecard-tests/bundle/tests/scorecard/config.yaml b/internal/scorecard/examples/custom-scorecard-tests/bundle/tests/scorecard/config.yaml index e1cc333f526..9d34e5aebd8 100644 --- a/internal/scorecard/examples/custom-scorecard-tests/bundle/tests/scorecard/config.yaml +++ b/internal/scorecard/examples/custom-scorecard-tests/bundle/tests/scorecard/config.yaml @@ -1,4 +1,4 @@ -kind: ScorecardConfiguration +kind: Configuration apiversion: scorecard.operatorframework.io/v1alpha3 metadata: name: config diff --git a/internal/scorecard/formatting.go b/internal/scorecard/formatting.go index 6e2778cc8c1..96a7b79afb6 100644 --- a/internal/scorecard/formatting.go +++ b/internal/scorecard/formatting.go @@ -46,9 +46,7 @@ func (o Scorecard) List() v1alpha3.TestList { tests := o.selectTests(stage) for _, test := range tests { item := v1alpha3.NewTest() - item.Spec.Image = test.Image - item.Spec.Entrypoint = test.Entrypoint - item.Spec.Labels = test.Labels + item.Spec = test output.Items = append(output.Items, item) } } diff --git a/internal/scorecard/scorecard.go b/internal/scorecard/scorecard.go index 03f731d433c..f1c9f876842 100644 --- a/internal/scorecard/scorecard.go +++ b/internal/scorecard/scorecard.go @@ -118,9 +118,7 @@ func (o Scorecard) runTest(ctx context.Context, test v1alpha3.TestConfiguration) } out := v1alpha3.NewTest() - out.Spec.Image = test.Image - out.Spec.Entrypoint = test.Entrypoint - out.Spec.Labels = test.Labels + out.Spec = test out.Status = *result return out } diff --git a/internal/scorecard/testdata/bundle.tar.gz b/internal/scorecard/testdata/bundle.tar.gz index 081bf80cd41ed92615bfe81c6298b467f5221a06..5f67c14017abcea5aee95567d6c422ee493df1e3 100644 GIT binary patch literal 2762 zcmV;*3N`f~iwFR;?ipVI1MM7LbK|zr&;At{_rWvmkrJ(w)ap&Ooz&B_t+CwnA(Od* zNJv5rejEtOr=K} zTmH=8$mtEcgI;gY?G2B*&RO3X9+Cdru~a}3D=G;&0vX8g=k4QmJ^!78pX=XezDwOD z^Q zAXHQVI>w)X5WdB}Qa=(b9+CfA8cE2H=_et-H*gSo4ADU% zM0s-YHOKy%e)~y(x-_CQF1X?-*T2)4yW}c>Wi%b!*oZVZ^{Q)kZ6_TON7RL3k${h7 zC_E;QaeG>i5l^nDQgU5|tq|AZQk5Y7v z<9px=5L^P!LE_JtB;mYRmT;7K;4;`w2m>P#uI*f59*;q^>l@AL6f7iv5>d4rk+_Wd zYI&R7AU~&x3`XEZqvH7pwr4BmsxaWLo$yf6BCqIi7z(^iHjw-=h- z!*GggZe`_ma(OqtzrA~spPj~lzIn#Gz8t$QhT@y2Ny0Cm$8nK0QARQmtP<`Z3x6-eB&ry=m4%x+OO-K| z@Qa9)y=G(wayS)`k7V^Z3%n@g5T>fMF>^p67^qb!5}&z(a=$il#p<)bUFUq^)2PmZ zD_LbIB&JzyH!uX^OaP?@>~)Q4W^RgGK3Rv0HShZKkt#Y`eapni!s|MIBdbyLc)( z?ACs>hc)b^Cg*P%1a`>nm8QPUcAH(`rMzsb^Pgbs^v|bHFZ_N!^NgHP)L?}`iyHG9 zi(XTh#z_=~Qo$LFD4CemO;LLL7~}=Rj-gZVR>8K8o>K@#rQz3!P<&t7VT{l9MC=?!ZAKW8xLwf(2ytMKm0YedDvMwGpOtH z_MdM2+@(?WB5(XuI)Hn0X0pTF0^|FrF{B4Y0&Z#mVeI{975^y<$B{@R6=fDo{Dg>> z2r5et03W*{hNL_mXlQV6cosTNu#(q9NaahjaY)O1&?%ct+&Hu?`& zKGsw;P$P1e1mq;E5UtpIfC4qZUyOi;PS6D<%TX@WhlI|XX!@bY=WAn}XjEJa%}vxOViMNW^5e>-sc&7CV{89Tzbv|!aU-IoNlQahEFoA%p` z(4>pqW~j$i3^APHk>I747-Xk_GVCblBXG-Dy9 z^BpKh>b9)Lymxim!Y;_t>Kk7FdCC2sUU$&m|2YKY`Y*j+dihDe8Frlioo@a7@AUg; z!}k1t2&moGDXsuz-{umc{$nT~AU)c%B=F#Xp1{8B{f7tj#&>@ao}d4n!LZxr|3g6C<=-M2eaC(E@S*!m;KwbZ7Hu$C$;Eww5tT%LW{X1s^tpA3CL0kVF0@BwyTI-24 z?SC5aycv*NMq>q$-Nx=Uy#FzlbPVEMU`GmHj!P;86M45l+n#AHq<6W#By^2r^9EAB zSYe_TP8)e_rTYymfXY*MzO(zr)qyEeH*_p0^4tdt8^q=nQsp{goNxs!=#g261XtRa zo6&uHu0!ElrcShqe%E%^VR(h{#_5OU-j!Z#E$Qmz8)%~NUVF%26TNf`If29OKwE9; zKK1k1ug^X(@47?&3hp=Nn-$(W@cu@ip8p>brNlR`0-s<1_Yu|Be+PhO{o~~d3tw7% z6H9o${s%*+?f)JGcIm&{#n0EB2X@r|T{!gB?tgXR0MOR|hk)|^uT^%-3;#xJv!Dk% zu&}9EPQ5gWMkHA9;A?)RtNJ%N*;_+12B)M&gGP)Tduacy&BgBrd-y+%Vpw3oW$JLJKXl Q&_WIV2gG!XO#n~;0J-^lGXMYp literal 2768 zcmV;>3NQ5^iwFSJ!5Civ1MM8ka^twsXMY9CIk+krQZHLdbW?2a#uYzQma{o*ZB0QW zBq0I`4oJ$Q-R!@o8{k`%CEH^;yURX|q$bd4^czihJnHiq3!|`g^ls1w7!1zwvoknv z7e7-t>huQfetU4%8}yIbo$h(}?1-GdA4@sJVMG-nM<4?wf8IWB*Yn>fc$xk^<~h_^ zFxQd+Q#6vwb_7U=sgS7^ci0NPap~J@>WLSvfeP;T&$j5l)9&>KW&L-1{ciV&wBKpZ zCHRBup9cH~qt_ux^&Z-eR7~QB*IIR`2o|(+yyU(+Bp1dcCdMR^smC6Rs7oU{JSK$t zzKm!D=n#JbLUnl(?vQIAmeF)@V?$Eo)T_4LwmZp)FrW?$3j};D zq;Q!!#_ef620U?mm(94(jYi2Y$bdHI|o6O|X>>g0)s{B^kI%?K?ANgU5|tCd`4+4-<5S z<452M5L^Mze(X(|B65~5OE^m0cNlC(!oWbtwVf%<OT?b#7?BI$F-j`=WZY(QBgV0GGtvdgq%zS) zx?3uDXtabXrnROm3B^)UApxGkdQs@<6h= z6Yqf$+yM&5$jX$4tbeBC9yu}b!ecI{(K=uWh8ur08%RA%i$hzA$NZ=u;%z>gZNEst zXgLtvq2QLC0ybm_5Mn?w+1oC)`V&FY^f&&S#=Qd!!Cy!zv*6kzdG%~^DpM)C0Ur>zW+?=LmG zhv5|0+{()R`08Qwc>nM$KRb;9fqQfgBOk8DH=u;i*SDkZHwo%NV@R!I37!JxI}%TH z9lp*45%R!V_5_3^Uyd9HL-F17B;l9G!!Xa9C?gdMRtk5Jg}+xa4oU`YrQxQ^QfW*n z{30S@uO8Wg98Lw~BUOIRd^eCB!c>_yW)3I>{b(gc>@i1B?o}qPSalY->x|Dm8dO}$LBpsp*_c9zMuP{hWHM<4=ylmkqlGdo9zfOP9Zu)g2+ z_MdM2JS0)}GHd*lI)Hn0X41pl9OL`2Hl+J`0A@Sp=Y{T`5FJMoi7M(kN|Xp zFH0B&jTO>LX{}Ra#*87lImgVBNpJ)nLbJGT*#JL~xM9DLPH55w$pMB?pUCNdm=oCq z`@TSggbNu97Ylm$8Nqx<&V49tQp&m;hD%~|pp+yrUqwtoC1j3zpMbmzaeWO$a19ds z1!{UoXp>v08e!s04#^^lpdx6s<~&Mm6p#8b7~;Bx4FSccF_w!h7mgsJ74o@7m9ya3 zA&M1ifdDPdxUUb>w&(sSQ+BaTFsTaM9?T{~Yg61RcQi=UFHNQy{&Vpn3Z?RJHF?Zx z6|I7@%(b-hX<;sS6cosTDWkX<9Naa<^q8i9ID(tfAP2B$k4LEKFENID(2q?nbWedy zSyHCBHo5iz>oIWwRouIa9VHxEDAL`6uj?t9fks++K~?D!)qCLtMwNUc?eXe!7FvC^ z-@MHq^_pO;)ecG7GOq6?R}YV5ba{FGczu62x*?PC)dl(P`s(I#V!Es4<&$@B8~ulC z4{IvwM?>-u`{X385UtQ%LV@bzFGfH^C+LETWhe*gLqcbDG(G9^+1eN<8Wk6ej9fQq z^B2M$i7|$fWjtlmnCtdwMCUafz0_K!QZB7k$Hq^)Fb&OT%GT0%KTq++Mc7LC1oFDr zSVW%Kpo$&zG<>4SH(SnmVSHMd4RIEV$00_aUbphh*b&C1IV+#(zQm}N%Bi&2wBKfg zCSB|{Lp`o!h;oWYg4bH2pPmAWu+tPDN00xS7-Q77h}={d*-%Dqqp!Bf4Vt-)zQrat z$Z8vXC!5^+O-R)sAuxJ1o>LqAijSyLbiLC^%WqR^t+axwdrg+$k`tJz>q>9YjD-}= zcc2`l+p-$-(ba7OyC6-g?|A*^HTQpdonCYQ=Ma$TzwmnL^(Xy$*m3^vw5#X;&iQ%o ztU3Q50xGw4@+&~;x0!;d|0LBC(xW|%eHRYs*;{UV)fr+whl96T|3%*-*pouqcG48` zCe#a9NM1JT`Kb}_Jsl4EOGs16?3!-teq@{l$d3uL3!N4}fjLbDdnpAQYAza!NE6zZ z1fHxbx(M<0ibIvVmohW~^HNUFjT;T~g^@Vi@xXSkNQW zRtc`OF*l|A{#;ApS*BpL^2XP8)?s*w@!IK!<=#om{s!(h=9?8hIs$(zRP+Bbjw11mtH77n|L5&?Q~w_X>h+J8 zD=d6z@l7n@<@)arx=sK0Ah1jS?KXbC?L4rf{%?1|*~|C8;A^j`{|^Dh`(LZ{mKXkw z*k&CMc3@#sv6yjb5DbYw=l<91N>}-Ba?-bkW(-bAY6XoLcI<)uw?B9a{!rM%|7jR9 z6}=$@?DBsI_tpEKys%u*SPyhfolV`yI diff --git a/internal/scorecard/testdata/bundle/tests/scorecard/config.yaml b/internal/scorecard/testdata/bundle/tests/scorecard/config.yaml index 0a8ac76302f..6e693a59164 100644 --- a/internal/scorecard/testdata/bundle/tests/scorecard/config.yaml +++ b/internal/scorecard/testdata/bundle/tests/scorecard/config.yaml @@ -1,4 +1,4 @@ -kind: ScorecardConfiguration +kind: Configuration apiversion: scorecard.operatorframework.io/v1alpha3 metadata: name: config diff --git a/pkg/apis/scorecard/v1alpha3/configuration_types.go b/pkg/apis/scorecard/v1alpha3/configuration_types.go index 0228eb8d1a1..631f46c8dce 100644 --- a/pkg/apis/scorecard/v1alpha3/configuration_types.go +++ b/pkg/apis/scorecard/v1alpha3/configuration_types.go @@ -19,7 +19,7 @@ import ( ) // ConfigurationKind is the default scorecard componentconfig kind. -const ConfigurationKind = "ScorecardConfiguration" +const ConfigurationKind = "Configuration" // Configuration represents the set of test configurations which scorecard would run. type Configuration struct { @@ -27,6 +27,7 @@ type Configuration struct { // Do not use metav1.ObjectMeta because this "object" should not be treated as an actual object. Metadata struct { + // Name is a required field for kustomize-able manifests, and is not used on-cluster (nor is the config itself). Name string `json:"name,omitempty" yaml:"name,omitempty"` } `json:"metadata,omitempty" yaml:"metadata,omitempty"` diff --git a/pkg/apis/scorecard/v1alpha3/test_types.go b/pkg/apis/scorecard/v1alpha3/test_types.go index 2096792e867..4480462ad5a 100644 --- a/pkg/apis/scorecard/v1alpha3/test_types.go +++ b/pkg/apis/scorecard/v1alpha3/test_types.go @@ -30,16 +30,6 @@ const ( ErrorState State = "error" ) -// TestSpec contains the spec details of an individual scorecard test -type TestSpec struct { - // Image is the name of the testimage - Image string `json:"image"` - // Entrypoint is list of commands and arguments passed to the test image - Entrypoint []string `json:"entrypoint,omitempty"` - // Labels that further describe the test and enable selection - Labels map[string]string `json:"labels,omitempty"` -} - // TestResult contains the results of an individual scorecard test type TestResult struct { // Name is the name of the test @@ -64,8 +54,8 @@ type TestStatus struct { // Test specifies a single test run. type Test struct { metav1.TypeMeta `json:",inline"` - Spec TestSpec `json:"spec,omitempty"` - Status TestStatus `json:"status,omitempty"` + Spec TestConfiguration `json:"spec,omitempty"` + Status TestStatus `json:"status,omitempty"` } // TestList is a list of tests. diff --git a/pkg/apis/scorecard/v1alpha3/zz_generated.deepcopy.go b/pkg/apis/scorecard/v1alpha3/zz_generated.deepcopy.go index 0ba9be02e93..38350de867e 100644 --- a/pkg/apis/scorecard/v1alpha3/zz_generated.deepcopy.go +++ b/pkg/apis/scorecard/v1alpha3/zz_generated.deepcopy.go @@ -22,12 +22,61 @@ import ( runtime "k8s.io/apimachinery/pkg/runtime" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Configuration) DeepCopyInto(out *Configuration) { + *out = *in + out.TypeMeta = in.TypeMeta + out.Metadata = in.Metadata + if in.Stages != nil { + in, out := &in.Stages, &out.Stages + *out = make([]StageConfiguration, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Configuration. +func (in *Configuration) DeepCopy() *Configuration { + if in == nil { + return nil + } + out := new(Configuration) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StageConfiguration) DeepCopyInto(out *StageConfiguration) { + *out = *in + if in.Tests != nil { + in, out := &in.Tests, &out.Tests + *out = make([]TestConfiguration, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StageConfiguration. +func (in *StageConfiguration) DeepCopy() *StageConfiguration { + if in == nil { + return nil + } + out := new(StageConfiguration) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Test) DeepCopyInto(out *Test) { *out = *in out.TypeMeta = in.TypeMeta in.Spec.DeepCopyInto(&out.Spec) in.Status.DeepCopyInto(&out.Status) + return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Test. @@ -48,6 +97,34 @@ func (in *Test) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TestConfiguration) DeepCopyInto(out *TestConfiguration) { + *out = *in + if in.Entrypoint != nil { + in, out := &in.Entrypoint, &out.Entrypoint + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Labels != nil { + in, out := &in.Labels, &out.Labels + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TestConfiguration. +func (in *TestConfiguration) DeepCopy() *TestConfiguration { + if in == nil { + return nil + } + out := new(TestConfiguration) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TestList) DeepCopyInto(out *TestList) { *out = *in @@ -59,6 +136,7 @@ func (in *TestList) DeepCopyInto(out *TestList) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TestList. @@ -84,6 +162,7 @@ func (in *TestResult) DeepCopyInto(out *TestResult) { *out = make([]string, len(*in)) copy(*out, *in) } + return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TestResult. @@ -96,33 +175,6 @@ func (in *TestResult) DeepCopy() *TestResult { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *TestSpec) DeepCopyInto(out *TestSpec) { - *out = *in - if in.Entrypoint != nil { - in, out := &in.Entrypoint, &out.Entrypoint - *out = make([]string, len(*in)) - copy(*out, *in) - } - if in.Labels != nil { - in, out := &in.Labels, &out.Labels - *out = make(map[string]string, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TestSpec. -func (in *TestSpec) DeepCopy() *TestSpec { - if in == nil { - return nil - } - out := new(TestSpec) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TestStatus) DeepCopyInto(out *TestStatus) { *out = *in @@ -133,6 +185,7 @@ func (in *TestStatus) DeepCopyInto(out *TestStatus) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TestStatus. diff --git a/website/content/en/docs/advanced-topics/scorecard/custom-tests.md b/website/content/en/docs/advanced-topics/scorecard/custom-tests.md index 13ccc3d7d44..463791ff2b6 100644 --- a/website/content/en/docs/advanced-topics/scorecard/custom-tests.md +++ b/website/content/en/docs/advanced-topics/scorecard/custom-tests.md @@ -101,7 +101,7 @@ For the example `CustomTest1` function, add the following to `config/scorecard/c ```yaml - op: add - path: /stages/0/tests/0 + path: /stages/0/tests/- value: image: quay.io//custom-scorecard-tests:latest entrypoint: @@ -126,7 +126,7 @@ patchesJson6902: target: group: scorecard.operatorframework.io version: v1alpha3 - kind: ScorecardConfiguration + kind: Configuration name: config ``` @@ -206,16 +206,32 @@ The `operator-sdk scorecard` command is used to execute the scorecard tests by s ```console $ operator-sdk scorecard --selector=suite=custom -o json --wait-time=32s --skip-cleanup=false { - "metadata": { - "creationTimestamp": null - }, - "log": "", - "results": [ + "kind": "TestList", + "apiVersion": "scorecard.operatorframework.io/v1alpha3", + "items": [ { - "name": "customtest1", - "description": "", - "state": "pass", - "log": "an ISV custom test" + "kind": "Test", + "apiVersion": "scorecard.operatorframework.io/v1alpha3", + "spec": { + "image": "quay.io/operator-framework/scorecard-test:latest", + "entrypoint": [ + "custom-scorecard-tests", + "customtest1" + ], + "labels": { + "suite": "custom", + "test": "customtest1" + } + }, + "status": { + "results": [ + { + "name": "customtest1", + "log": "an ISV custom test", + "state": "pass" + } + ] + } } ] } diff --git a/website/content/en/docs/advanced-topics/scorecard/scorecard.md b/website/content/en/docs/advanced-topics/scorecard/scorecard.md index 7fa6b14f1e4..1f10db9699f 100644 --- a/website/content/en/docs/advanced-topics/scorecard/scorecard.md +++ b/website/content/en/docs/advanced-topics/scorecard/scorecard.md @@ -67,7 +67,7 @@ $ tree ./bundle The following YAML spec is an example of the scorecard configuration file: ```yaml -kind: ScorecardConfiguration +kind: Configuration apiversion: scorecard.operatorframework.io/v1alpha3 metadata: name: config