wip: more progress

cmd/readmevalidation/coderResources.go

@@ -10,6 +10,7 @@ import (
 	"strings"
 
 	"coder.com/coder-registry/cmd/github"
+	"gopkg.in/yaml.v3"
 )
 
 // dummyGitDirectory is the directory that a full version of the Registry will
@@ -34,7 +35,7 @@ type coderResourceFrontmatter struct {
 // Coder Modules and Coder Templates. If the newReadmeBody and newFrontmatter
 // fields are nil, that represents that the file has been deleted
 type coderResource struct {
-	name           string
+	resourceType   string
 	filePath       string
 	newReadmeBody  *string
 	oldFrontmatter *coderResourceFrontmatter
@@ -178,10 +179,77 @@ func validateCoderResourceChanges(resource coderResource, actorOrgStatus github.
 	return problems
 }
 
-func parseCoderResourceFiles(oldReadmeFiles []readme, newReadmeFiles []readme, actorOrgStatus github.OrgStatus) (map[string]coderResource, error) {
-	return nil, nil
+func parseCoderResourceFiles(resourceType string, oldReadmeFiles []readme, newReadmeFiles []readme, actorOrgStatus github.OrgStatus) (map[string]coderResource, error) {
+	if !slices.Contains(supportedResourceTypes, resourceType) {
+		return nil, fmt.Errorf("resource type %q is not in supported list [%s]", resourceType, strings.Join(supportedResourceTypes, ", "))
+	}
+
+	var errs []error
+	resourcesByFilePath := map[string]coderResource{}
+	zipped := zipReadmes(oldReadmeFiles, newReadmeFiles)
+
+	for filePath, z := range zipped {
+		resource := coderResource{
+			resourceType: resourceType,
+			filePath:     filePath,
+		}
+
+		if z.new != nil {
+			fm, body, err := separateFrontmatter(z.new.rawText)
+			if err != nil {
+				errs = append(errs, fmt.Errorf("resource type %s - %q: %v", resourceType, filePath, err))
+			} else {
+				resource.newReadmeBody = &body
+				var newFm coderResourceFrontmatter
+				if err := yaml.Unmarshal([]byte(fm), &newFm); err != nil {
+					errs = append(errs, fmt.Errorf("resource type %s - %q: %v", resourceType, filePath, err))
+				} else {
+					resource.newFrontmatter = &newFm
+					if newFm.Verified != nil && *newFm.Verified {
+						resource.newIsVerified = true
+					}
+				}
+			}
+		}
+
+		if z.old != nil {
+			fm, _, err := separateFrontmatter(z.old.rawText)
+			if err != nil {
+				errs = append(errs, fmt.Errorf("resource type %s - %q: %v", resourceType, filePath, err))
+			} else {
+				var oldFm coderResourceFrontmatter
+				if err := yaml.Unmarshal([]byte(fm), &oldFm); err != nil {
+					errs = append(errs, fmt.Errorf("resource type %s - %q: %v", resourceType, filePath, err))
+				} else {
+					resource.oldFrontmatter = &oldFm
+					if oldFm.Verified != nil && *oldFm.Verified {
+						resource.oldIsVerified = true
+					}
+				}
+			}
+		}
+
+		if z.old != nil || z.new != nil {
+			resourcesByFilePath[filePath] = resource
+		}
+	}
+
+	for _, r := range resourcesByFilePath {
+		errs = append(errs, validateCoderResourceChanges(r, actorOrgStatus)...)
+	}
+
+	if len(errs) != 0 {
+		return nil, validationPhaseError{
+			phase:  validationPhaseReadmeParsing,
+			errors: errs,
+		}
+	}
+	return resourcesByFilePath, nil
 }
 
+// Todo: because Coder Resource READMEs will have their full contents
+// (frontmatter and body) rendered on the Registry site, we need to make sure
+// that all image references in the body are valid, too
 func validateCoderResourceRelativeUrls(map[string]coderResource) []error {
 	return nil
 }

cmd/readmevalidation/main.go

@@ -13,9 +13,6 @@ import (
 	"sync"
 
 	"coder.com/coder-registry/cmd/github"
-	"github.com/go-git/go-git/v5"
-	"github.com/go-git/go-git/v5/plumbing"
-	"github.com/go-git/go-git/v5/plumbing/transport/http"
 	"github.com/joho/godotenv"
 )
 
@@ -104,53 +101,56 @@ func main() {
 	go func() {
 		defer wg.Done()
 
-		refactorLater := func() error {
+		moveToOuterScopeLater := func() error {
 			baseRefReadmeFiles, err := aggregateCoderResourceReadmeFiles("modules")
 			if err != nil {
 				return err
 			}
-			fmt.Printf("------ got %d back\n", len(baseRefReadmeFiles))
-
-			repo, err := git.PlainClone(dummyGitDirectory, true, &git.CloneOptions{
-				URL:  "https://github.com/coder/registry",
-				Auth: &http.BasicAuth{},
-			})
+			parsed, err := parseCoderResourceFiles("modules", baseRefReadmeFiles, baseRefReadmeFiles, actorOrgStatus)
 			if err != nil {
 				return err
 			}
+			fmt.Printf("------ got %d back\n", len(parsed))
 
-			head, err := repo.Head()
-			if err != nil {
-				return err
-			}
-			activeBranchName := head.Name().Short()
+			// repo, err := git.PlainClone(dummyGitDirectory, true, &git.CloneOptions{
+			// 	URL:  "https://github.com/coder/registry",
+			// 	Auth: &http.BasicAuth{},
+			// })
+			// if err != nil {
+			// 	return err
+			// }
 
-			tree, err := repo.Worktree()
-			if err != nil {
-				return err
-			}
-			err = tree.Checkout(&git.CheckoutOptions{
-				Branch: plumbing.NewBranchReferenceName(activeBranchName),
-				Create: false,
-				Force:  false,
-				Keep:   true,
-			})
-			if err != nil {
-				return err
-			}
+			// head, err := repo.Head()
+			// if err != nil {
+			// 	return err
+			// }
+			// activeBranchName := head.Name().Short()
 
-			fmt.Println("Got here!")
-			files, _ := tree.Filesystem.ReadDir(".")
-			for _, f := range files {
-				if f.IsDir() {
-					fmt.Println(f.Name())
-				}
-			}
+			// tree, err := repo.Worktree()
+			// if err != nil {
+			// 	return err
+			// }
+			// err = tree.Checkout(&git.CheckoutOptions{
+			// 	Branch: plumbing.NewBranchReferenceName(activeBranchName),
+			// 	Create: false,
+			// 	Force:  false,
+			// 	Keep:   true,
+			// })
+			// if err != nil {
+			// 	return err
+			// }
+
+			// files, _ := tree.Filesystem.ReadDir(".")
+			// for _, f := range files {
+			// 	if f.IsDir() {
+			// 		fmt.Println(f.Name())
+			// 	}
+			// }
 
 			return nil
 		}
 
-		if err := refactorLater(); err != nil {
+		if err := moveToOuterScopeLater(); err != nil {
 			errChan <- fmt.Errorf("module validation: %v", err)
 		}
 

cmd/readmevalidation/readmes.go

@@ -108,3 +108,48 @@ func (p validationPhase) String() string {
 		return "Unknown validation phase"
 	}
 }
+
+type zippedReadmes struct {
+	old *readme
+	new *readme
+}
+
+// zipReadmes takes two slices of README files, and combines them into a map,
+// where each key is a file path, and each value is a struct containing the old
+// value for the path, and the new value for the path. If the old value exists
+// but the new one doesn't, that indicates that a file has been deleted. If the
+// new value exists, but the old one doesn't, that indicates that the file was
+// created.
+func zipReadmes(prevReadmes []readme, newReadmes []readme) map[string]zippedReadmes {
+	oldMap := map[string]readme{}
+	for _, rm := range prevReadmes {
+		oldMap[rm.filePath] = rm
+	}
+
+	zipped := map[string]zippedReadmes{}
+	for _, rm := range newReadmes {
+		old, ok := oldMap[rm.filePath]
+		if ok {
+			zipped[rm.filePath] = zippedReadmes{
+				old: &old,
+				new: &rm,
+			}
+		} else {
+			zipped[rm.filePath] = zippedReadmes{
+				old: nil,
+				new: &rm,
+			}
+		}
+	}
+	for _, old := range oldMap {
+		_, ok := zipped[old.filePath]
+		if !ok {
+			zipped[old.filePath] = zippedReadmes{
+				old: &old,
+				new: nil,
+			}
+		}
+	}
+
+	return zipped
+}

cmd/readmevalidation/repoStructure.go

@@ -37,15 +37,24 @@ func validateCoderResourceSubdirectory(dirPath string) []error {
 
 		resourceReadmePath := path.Join(dirPath, f.Name(), "README.md")
 		_, err := os.Stat(resourceReadmePath)
-		if err == nil {
-			continue
+		if err != nil {
+			if errors.Is(err, os.ErrNotExist) {
+				errs = append(errs, fmt.Errorf("%q: README file does not exist", resourceReadmePath))
+			} else {
+				errs = append(errs, addFilePathToError(resourceReadmePath, err))
+			}
 		}
 
-		if errors.Is(err, os.ErrNotExist) {
-			errs = append(errs, fmt.Errorf("%q: README file does not exist", resourceReadmePath))
-		} else {
-			errs = append(errs, addFilePathToError(resourceReadmePath, err))
+		mainTerraformPath := path.Join(dirPath, f.Name(), "main.tf")
+		_, err = os.Stat(mainTerraformPath)
+		if err != nil {
+			if errors.Is(err, os.ErrNotExist) {
+				errs = append(errs, fmt.Errorf("%q: 'main.tf' file does not exist", mainTerraformPath))
+			} else {
+				errs = append(errs, addFilePathToError(mainTerraformPath, err))
+			}
 		}
+
 	}
 
 	return errs
@@ -71,12 +80,6 @@ func validateRegistryDirectory() []error {
 			problems = append(problems, err)
 		}
 
-		mainTerraformPath := path.Join(dirPath, "main.tf")
-		_, err = os.Stat(mainTerraformPath)
-		if err != nil {
-			problems = append(problems, err)
-		}
-
 		for _, rType := range supportedResourceTypes {
 			resourcePath := path.Join(dirPath, rType)
 			if errs := validateCoderResourceSubdirectory(resourcePath); len(errs) != 0 {