From e6efd71fcafa6c442ca134f88ac0a1bfc1ff59c5 Mon Sep 17 00:00:00 2001 From: Michael Smith Date: Wed, 16 Apr 2025 04:02:45 +0000 Subject: [PATCH] wip: more progress --- cmd/readmevalidation/coderResources.go | 74 ++++++++++++++++++++++++-- cmd/readmevalidation/main.go | 72 ++++++++++++------------- cmd/readmevalidation/readmes.go | 45 ++++++++++++++++ cmd/readmevalidation/repoStructure.go | 27 +++++----- 4 files changed, 167 insertions(+), 51 deletions(-) diff --git a/cmd/readmevalidation/coderResources.go b/cmd/readmevalidation/coderResources.go index eda5fd6f..d4661c59 100644 --- a/cmd/readmevalidation/coderResources.go +++ b/cmd/readmevalidation/coderResources.go @@ -10,6 +10,7 @@ import ( "strings" "coder.com/coder-registry/cmd/github" + "gopkg.in/yaml.v3" ) // dummyGitDirectory is the directory that a full version of the Registry will @@ -34,7 +35,7 @@ type coderResourceFrontmatter struct { // Coder Modules and Coder Templates. If the newReadmeBody and newFrontmatter // fields are nil, that represents that the file has been deleted type coderResource struct { - name string + resourceType string filePath string newReadmeBody *string oldFrontmatter *coderResourceFrontmatter @@ -178,10 +179,77 @@ func validateCoderResourceChanges(resource coderResource, actorOrgStatus github. return problems } -func parseCoderResourceFiles(oldReadmeFiles []readme, newReadmeFiles []readme, actorOrgStatus github.OrgStatus) (map[string]coderResource, error) { - return nil, nil +func parseCoderResourceFiles(resourceType string, oldReadmeFiles []readme, newReadmeFiles []readme, actorOrgStatus github.OrgStatus) (map[string]coderResource, error) { + if !slices.Contains(supportedResourceTypes, resourceType) { + return nil, fmt.Errorf("resource type %q is not in supported list [%s]", resourceType, strings.Join(supportedResourceTypes, ", ")) + } + + var errs []error + resourcesByFilePath := map[string]coderResource{} + zipped := zipReadmes(oldReadmeFiles, newReadmeFiles) + + for filePath, z := range zipped { + resource := coderResource{ + resourceType: resourceType, + filePath: filePath, + } + + if z.new != nil { + fm, body, err := separateFrontmatter(z.new.rawText) + if err != nil { + errs = append(errs, fmt.Errorf("resource type %s - %q: %v", resourceType, filePath, err)) + } else { + resource.newReadmeBody = &body + var newFm coderResourceFrontmatter + if err := yaml.Unmarshal([]byte(fm), &newFm); err != nil { + errs = append(errs, fmt.Errorf("resource type %s - %q: %v", resourceType, filePath, err)) + } else { + resource.newFrontmatter = &newFm + if newFm.Verified != nil && *newFm.Verified { + resource.newIsVerified = true + } + } + } + } + + if z.old != nil { + fm, _, err := separateFrontmatter(z.old.rawText) + if err != nil { + errs = append(errs, fmt.Errorf("resource type %s - %q: %v", resourceType, filePath, err)) + } else { + var oldFm coderResourceFrontmatter + if err := yaml.Unmarshal([]byte(fm), &oldFm); err != nil { + errs = append(errs, fmt.Errorf("resource type %s - %q: %v", resourceType, filePath, err)) + } else { + resource.oldFrontmatter = &oldFm + if oldFm.Verified != nil && *oldFm.Verified { + resource.oldIsVerified = true + } + } + } + } + + if z.old != nil || z.new != nil { + resourcesByFilePath[filePath] = resource + } + } + + for _, r := range resourcesByFilePath { + errs = append(errs, validateCoderResourceChanges(r, actorOrgStatus)...) + } + + if len(errs) != 0 { + return nil, validationPhaseError{ + phase: validationPhaseReadmeParsing, + errors: errs, + } + } + return resourcesByFilePath, nil } +// Todo: because Coder Resource READMEs will have their full contents +// (frontmatter and body) rendered on the Registry site, we need to make sure +// that all image references in the body are valid, too func validateCoderResourceRelativeUrls(map[string]coderResource) []error { return nil } diff --git a/cmd/readmevalidation/main.go b/cmd/readmevalidation/main.go index f58fc47d..56ee3824 100644 --- a/cmd/readmevalidation/main.go +++ b/cmd/readmevalidation/main.go @@ -13,9 +13,6 @@ import ( "sync" "coder.com/coder-registry/cmd/github" - "github.com/go-git/go-git/v5" - "github.com/go-git/go-git/v5/plumbing" - "github.com/go-git/go-git/v5/plumbing/transport/http" "github.com/joho/godotenv" ) @@ -104,53 +101,56 @@ func main() { go func() { defer wg.Done() - refactorLater := func() error { + moveToOuterScopeLater := func() error { baseRefReadmeFiles, err := aggregateCoderResourceReadmeFiles("modules") if err != nil { return err } - fmt.Printf("------ got %d back\n", len(baseRefReadmeFiles)) - - repo, err := git.PlainClone(dummyGitDirectory, true, &git.CloneOptions{ - URL: "https://github.com/coder/registry", - Auth: &http.BasicAuth{}, - }) + parsed, err := parseCoderResourceFiles("modules", baseRefReadmeFiles, baseRefReadmeFiles, actorOrgStatus) if err != nil { return err } + fmt.Printf("------ got %d back\n", len(parsed)) - head, err := repo.Head() - if err != nil { - return err - } - activeBranchName := head.Name().Short() + // repo, err := git.PlainClone(dummyGitDirectory, true, &git.CloneOptions{ + // URL: "https://github.com/coder/registry", + // Auth: &http.BasicAuth{}, + // }) + // if err != nil { + // return err + // } - tree, err := repo.Worktree() - if err != nil { - return err - } - err = tree.Checkout(&git.CheckoutOptions{ - Branch: plumbing.NewBranchReferenceName(activeBranchName), - Create: false, - Force: false, - Keep: true, - }) - if err != nil { - return err - } + // head, err := repo.Head() + // if err != nil { + // return err + // } + // activeBranchName := head.Name().Short() - fmt.Println("Got here!") - files, _ := tree.Filesystem.ReadDir(".") - for _, f := range files { - if f.IsDir() { - fmt.Println(f.Name()) - } - } + // tree, err := repo.Worktree() + // if err != nil { + // return err + // } + // err = tree.Checkout(&git.CheckoutOptions{ + // Branch: plumbing.NewBranchReferenceName(activeBranchName), + // Create: false, + // Force: false, + // Keep: true, + // }) + // if err != nil { + // return err + // } + + // files, _ := tree.Filesystem.ReadDir(".") + // for _, f := range files { + // if f.IsDir() { + // fmt.Println(f.Name()) + // } + // } return nil } - if err := refactorLater(); err != nil { + if err := moveToOuterScopeLater(); err != nil { errChan <- fmt.Errorf("module validation: %v", err) } diff --git a/cmd/readmevalidation/readmes.go b/cmd/readmevalidation/readmes.go index 00bcca18..78de0e41 100644 --- a/cmd/readmevalidation/readmes.go +++ b/cmd/readmevalidation/readmes.go @@ -108,3 +108,48 @@ func (p validationPhase) String() string { return "Unknown validation phase" } } + +type zippedReadmes struct { + old *readme + new *readme +} + +// zipReadmes takes two slices of README files, and combines them into a map, +// where each key is a file path, and each value is a struct containing the old +// value for the path, and the new value for the path. If the old value exists +// but the new one doesn't, that indicates that a file has been deleted. If the +// new value exists, but the old one doesn't, that indicates that the file was +// created. +func zipReadmes(prevReadmes []readme, newReadmes []readme) map[string]zippedReadmes { + oldMap := map[string]readme{} + for _, rm := range prevReadmes { + oldMap[rm.filePath] = rm + } + + zipped := map[string]zippedReadmes{} + for _, rm := range newReadmes { + old, ok := oldMap[rm.filePath] + if ok { + zipped[rm.filePath] = zippedReadmes{ + old: &old, + new: &rm, + } + } else { + zipped[rm.filePath] = zippedReadmes{ + old: nil, + new: &rm, + } + } + } + for _, old := range oldMap { + _, ok := zipped[old.filePath] + if !ok { + zipped[old.filePath] = zippedReadmes{ + old: &old, + new: nil, + } + } + } + + return zipped +} diff --git a/cmd/readmevalidation/repoStructure.go b/cmd/readmevalidation/repoStructure.go index ebc88097..fe1d9051 100644 --- a/cmd/readmevalidation/repoStructure.go +++ b/cmd/readmevalidation/repoStructure.go @@ -37,15 +37,24 @@ func validateCoderResourceSubdirectory(dirPath string) []error { resourceReadmePath := path.Join(dirPath, f.Name(), "README.md") _, err := os.Stat(resourceReadmePath) - if err == nil { - continue + if err != nil { + if errors.Is(err, os.ErrNotExist) { + errs = append(errs, fmt.Errorf("%q: README file does not exist", resourceReadmePath)) + } else { + errs = append(errs, addFilePathToError(resourceReadmePath, err)) + } } - if errors.Is(err, os.ErrNotExist) { - errs = append(errs, fmt.Errorf("%q: README file does not exist", resourceReadmePath)) - } else { - errs = append(errs, addFilePathToError(resourceReadmePath, err)) + mainTerraformPath := path.Join(dirPath, f.Name(), "main.tf") + _, err = os.Stat(mainTerraformPath) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + errs = append(errs, fmt.Errorf("%q: 'main.tf' file does not exist", mainTerraformPath)) + } else { + errs = append(errs, addFilePathToError(mainTerraformPath, err)) + } } + } return errs @@ -71,12 +80,6 @@ func validateRegistryDirectory() []error { problems = append(problems, err) } - mainTerraformPath := path.Join(dirPath, "main.tf") - _, err = os.Stat(mainTerraformPath) - if err != nil { - problems = append(problems, err) - } - for _, rType := range supportedResourceTypes { resourcePath := path.Join(dirPath, rType) if errs := validateCoderResourceSubdirectory(resourcePath); len(errs) != 0 {