yamanin/registry
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: use coder/slog + minor go style changes (#107)

Browse Source 
Changes are broken down in to multiples commits to hopefully make
reviewing easy. 1 commit for the slog change and then a commit per Go
file for style changes.

Style changes are generally:
- try to use full sentences for all comments
- try to stick to 120 column lines (not strict) instead of 80
- try to one line as many `call function, check if err != nil` blocks as
possible (ex: only err or variables are not reused outside the if statement)
- try to use `err` or `errs` for all return type names, previously used
`problems` in some cases but `errs` in others
- some minor readability changes
- `Todo` -> `TODO`, sometimes also useful to do `TODO (name):` to make
it easier to find things a specific author meant to follow up on
- comments for types/functions should generally start with `//
FunctionName/TypeName ...`

---------

Signed-off-by: Callum Styan <callumstyan@gmail.com>
This commit is contained in:
Callum Styan 2025-06-02 12:23:55 -07:00 committed by GitHub
parent 8e051a8e2c
commit 13a25ff4af
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 172 additions and 203 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

93
cmd/readmevalidation/coderresources.go
View File

@ -2,8 +2,8 @@ package main
import ( import (
"bufio" "bufio"
"context"
"errors" "errors"
"log"
"net/url" "net/url"
"os" "os"
"path" "path"
@ -15,7 +15,14 @@ import (
"gopkg.in/yaml.v3" "gopkg.in/yaml.v3"
) )
var supportedResourceTypes = []string{"modules", "templates"} var (
supportedResourceTypes = []string{"modules", "templates"}
// TODO: This is a holdover from the validation logic used by the Coder Modules repo. It gives us some assurance, but
// realistically, we probably want to parse any Terraform code snippets, and make some deeper guarantees about how it's
// structured. Just validating whether it *can* be parsed as Terraform would be a big improvement.
terraformVersionRe = regexp.MustCompile(`^\s*\bversion\s+=`)
)
type coderResourceFrontmatter struct { type coderResourceFrontmatter struct {
Description string `yaml:"description"` Description string `yaml:"description"`
@ -49,36 +56,36 @@ func validateCoderResourceDescription(description string) error {
return nil return nil
} }
func validateCoderResourceIconURL(iconURL string) []error { func isPermittedRelativeURL(checkURL string) bool {
problems := []error{} // Would normally be skittish about having relative paths like this, but it should be safe because we have
// guarantees about the structure of the repo, and where this logic will run.
return strings.HasPrefix(checkURL, "./") || strings.HasPrefix(checkURL, "/") || strings.HasPrefix(checkURL, "../../../../.icons")
}
func validateCoderResourceIconURL(iconURL string) []error {
if iconURL == "" { if iconURL == "" {
problems = append(problems, xerrors.New("icon URL cannot be empty")) return []error{xerrors.New("icon URL cannot be empty")}
return problems
} }
isAbsoluteURL := !strings.HasPrefix(iconURL, ".") && !strings.HasPrefix(iconURL, "/") errs := []error{}
if isAbsoluteURL {
// If the URL does not have a relative path.
if !strings.HasPrefix(iconURL, ".") && !strings.HasPrefix(iconURL, "/") {
if _, err := url.ParseRequestURI(iconURL); err != nil { if _, err := url.ParseRequestURI(iconURL); err != nil {
problems = append(problems, xerrors.New("absolute icon URL is not correctly formatted")) errs = append(errs, xerrors.New("absolute icon URL is not correctly formatted"))
} }
if strings.Contains(iconURL, "?") { if strings.Contains(iconURL, "?") {
problems = append(problems, xerrors.New("icon URLs cannot contain query parameters")) errs = append(errs, xerrors.New("icon URLs cannot contain query parameters"))
} }
return problems return errs
} }
// Would normally be skittish about having relative paths like this, but it // If the URL has a relative path.
// should be safe because we have guarantees about the structure of the if !isPermittedRelativeURL(iconURL) {
// repo, and where this logic will run. errs = append(errs, xerrors.Errorf("relative icon URL %q must either be scoped to that module's directory, or the top-level /.icons directory (this can usually be done by starting the path with \"../../../.icons\")", iconURL))
isPermittedRelativeURL := strings.HasPrefix(iconURL, "./") ||
strings.HasPrefix(iconURL, "/") ||
strings.HasPrefix(iconURL, "../../../../.icons")
if !isPermittedRelativeURL {
problems = append(problems, xerrors.Errorf("relative icon URL %q must either be scoped to that module's directory, or the top-level /.icons directory (this can usually be done by starting the path with \"../../../.icons\")", iconURL))
} }
return problems return errs
} }
func validateCoderResourceTags(tags []string) error { func validateCoderResourceTags(tags []string) error {
@ -89,9 +96,8 @@ func validateCoderResourceTags(tags []string) error {
return nil return nil
} }
// All of these tags are used for the module/template filter controls in the // All of these tags are used for the module/template filter controls in the Registry site. Need to make sure they
// Registry site. Need to make sure they can all be placed in the browser // can all be placed in the browser URL without issue.
// URL without issue.
invalidTags := []string{} invalidTags := []string{}
for _, t := range tags { for _, t := range tags {
if t != url.QueryEscape(t) { if t != url.QueryEscape(t) {
@ -105,16 +111,11 @@ func validateCoderResourceTags(tags []string) error {
return nil return nil
} }
// Todo: This is a holdover from the validation logic used by the Coder Modules
// repo. It gives us some assurance, but realistically, we probably want to
// parse any Terraform code snippets, and make some deeper guarantees about how
// it's structured. Just validating whether it *can* be parsed as Terraform
// would be a big improvement.
var terraformVersionRe = regexp.MustCompile(`^\s*\bversion\s+=`)
func validateCoderResourceReadmeBody(body string) []error { func validateCoderResourceReadmeBody(body string) []error {
trimmed := strings.TrimSpace(body)
var errs []error var errs []error
trimmed := strings.TrimSpace(body)
// TODO: this may cause unexpected behavior since the errors slice may have a 0 length. Add a test.
errs = append(errs, validateReadmeBody(trimmed)...) errs = append(errs, validateReadmeBody(trimmed)...)
foundParagraph := false foundParagraph := false
@ -130,9 +131,8 @@ func validateCoderResourceReadmeBody(body string) []error {
lineNum++ lineNum++
nextLine := lineScanner.Text() nextLine := lineScanner.Text()
// Code assumes that invalid headers would've already been handled by // Code assumes that invalid headers would've already been handled by the base validation function, so we don't
// the base validation function, so we don't need to check deeper if the // need to check deeper if the first line isn't an h1.
// first line isn't an h1.
if lineNum == 1 { if lineNum == 1 {
if !strings.HasPrefix(nextLine, "# ") { if !strings.HasPrefix(nextLine, "# ") {
break break
@ -159,15 +159,13 @@ func validateCoderResourceReadmeBody(body string) []error {
continue continue
} }
// Code assumes that we can treat this case as the end of the "h1 // Code assumes that we can treat this case as the end of the "h1 section" and don't need to process any further lines.
// section" and don't need to process any further lines.
if lineNum > 1 && strings.HasPrefix(nextLine, "#") { if lineNum > 1 && strings.HasPrefix(nextLine, "#") {
break break
} }
// Code assumes that if we've reached this point, the only other options // Code assumes that if we've reached this point, the only other options are:
// are: (1) empty spaces, (2) paragraphs, (3) HTML, and (4) asset // (1) empty spaces, (2) paragraphs, (3) HTML, and (4) asset references made via [] syntax.
// references made via [] syntax.
trimmedLine := strings.TrimSpace(nextLine) trimmedLine := strings.TrimSpace(nextLine)
isParagraph := trimmedLine != "" && !strings.HasPrefix(trimmedLine, "![") && !strings.HasPrefix(trimmedLine, "<") isParagraph := trimmedLine != "" && !strings.HasPrefix(trimmedLine, "![") && !strings.HasPrefix(trimmedLine, "<")
foundParagraph = foundParagraph || isParagraph foundParagraph = foundParagraph || isParagraph
@ -250,7 +248,7 @@ func parseCoderResourceReadmeFiles(resourceType string, rms []readme) (map[strin
} }
if len(yamlParsingErrs) != 0 { if len(yamlParsingErrs) != 0 {
return nil, validationPhaseError{ return nil, validationPhaseError{
phase: validationPhaseReadmeParsing, phase: validationPhaseReadme,
errors: yamlParsingErrs, errors: yamlParsingErrs,
} }
} }
@ -264,7 +262,7 @@ func parseCoderResourceReadmeFiles(resourceType string, rms []readme) (map[strin
} }
if len(yamlValidationErrors) != 0 { if len(yamlValidationErrors) != 0 {
return nil, validationPhaseError{ return nil, validationPhaseError{
phase: validationPhaseReadmeParsing, phase: validationPhaseReadme,
errors: yamlValidationErrors, errors: yamlValidationErrors,
} }
} }
@ -274,7 +272,7 @@ func parseCoderResourceReadmeFiles(resourceType string, rms []readme) (map[strin
// Todo: Need to beef up this function by grabbing each image/video URL from // Todo: Need to beef up this function by grabbing each image/video URL from
// the body's AST. // the body's AST.
func validateCoderResourceRelativeUrls(_ map[string]coderResourceReadme) error { func validateCoderResourceRelativeURLs(_ map[string]coderResourceReadme) error {
return nil return nil
} }
@ -321,7 +319,7 @@ func aggregateCoderResourceReadmeFiles(resourceType string) ([]readme, error) {
if len(errs) != 0 { if len(errs) != 0 {
return nil, validationPhaseError{ return nil, validationPhaseError{
phase: validationPhaseFileLoad, phase: validationPhaseFile,
errors: errs, errors: errs,
} }
} }
@ -338,17 +336,16 @@ func validateAllCoderResourceFilesOfType(resourceType string) error {
return err return err
} }
log.Printf("Processing %d README files\n", len(allReadmeFiles)) logger.Info(context.Background(), "rocessing README files", "num_files", len(allReadmeFiles))
resources, err := parseCoderResourceReadmeFiles(resourceType, allReadmeFiles) resources, err := parseCoderResourceReadmeFiles(resourceType, allReadmeFiles)
if err != nil { if err != nil {
return err return err
} }
log.Printf("Processed %d README files as valid Coder resources with type %q", len(resources), resourceType) logger.Info(context.Background(), "rocessed README files as valid Coder resources", "num_files", len(resources), "type", resourceType)
err = validateCoderResourceRelativeUrls(resources) if err := validateCoderResourceRelativeURLs(resources); err != nil {
if err != nil {
return err return err
} }
log.Printf("All relative URLs for %s READMEs are valid\n", resourceType) logger.Info(context.Background(), "all relative URLs for READMEs are valid", "type", resourceType)
return nil return nil
} }

65
cmd/readmevalidation/contributors.go
View File

@ -1,7 +1,7 @@
package main package main
import ( import (
"log" "context"
"net/url" "net/url"
"os" "os"
"path" "path"
@ -50,6 +50,9 @@ func validateContributorLinkedinURL(linkedinURL *string) error {
return nil return nil
} }
// validateContributorSupportEmail does best effort validation of a contributors email address. We can't 100% validate
// that this is correct without actually sending an email, especially because some contributors are individual developers
// and we don't want to do that on every single run of the CI pipeline. The best we can do is verify the general structure.
func validateContributorSupportEmail(email *string) []error { func validateContributorSupportEmail(email *string) []error {
if email == nil { if email == nil {
return nil return nil
@ -57,10 +60,6 @@ func validateContributorSupportEmail(email *string) []error {
errs := []error{} errs := []error{}
// Can't 100% validate that this is correct without actually sending
// an email, and especially with some contributors being individual
// developers, we don't want to do that on every single run of the CI
// pipeline. Best we can do is verify the general structure.
username, server, ok := strings.Cut(*email, "@") username, server, ok := strings.Cut(*email, "@")
if !ok { if !ok {
errs = append(errs, xerrors.Errorf("email address %q is missing @ symbol", *email)) errs = append(errs, xerrors.Errorf("email address %q is missing @ symbol", *email))
@ -110,21 +109,18 @@ func validateContributorStatus(status string) error {
return nil return nil
} }
// Can't validate the image actually leads to a valid resource in a pure // Can't validate the image actually leads to a valid resource in a pure function, but can at least catch obvious problems.
// function, but can at least catch obvious problems.
func validateContributorAvatarURL(avatarURL *string) []error { func validateContributorAvatarURL(avatarURL *string) []error {
if avatarURL == nil { if avatarURL == nil {
return nil return nil
} }
errs := []error{}
if *avatarURL == "" { if *avatarURL == "" {
errs = append(errs, xerrors.New("avatar URL must be omitted or non-empty string")) return []error{xerrors.New("avatar URL must be omitted or non-empty string")}
return errs
} }
// Have to use .Parse instead of .ParseRequestURI because this is the errs := []error{}
// one field that's allowed to be a relative URL. // Have to use .Parse instead of .ParseRequestURI because this is the one field that's allowed to be a relative URL.
if _, err := url.Parse(*avatarURL); err != nil { if _, err := url.Parse(*avatarURL); err != nil {
errs = append(errs, xerrors.Errorf("URL %q is not a valid relative or absolute URL", *avatarURL)) errs = append(errs, xerrors.Errorf("URL %q is not a valid relative or absolute URL", *avatarURL))
} }
@ -132,7 +128,7 @@ func validateContributorAvatarURL(avatarURL *string) []error {
errs = append(errs, xerrors.New("avatar URL is not allowed to contain search parameters")) errs = append(errs, xerrors.New("avatar URL is not allowed to contain search parameters"))
} }
matched := false var matched bool
for _, ff := range supportedAvatarFileFormats { for _, ff := range supportedAvatarFileFormats {
matched = strings.HasSuffix(*avatarURL, ff) matched = strings.HasSuffix(*avatarURL, ff)
if matched { if matched {
@ -210,22 +206,21 @@ func parseContributorFiles(readmeEntries []readme) (map[string]contributorProfil
} }
if len(yamlParsingErrors) != 0 { if len(yamlParsingErrors) != 0 {
return nil, validationPhaseError{ return nil, validationPhaseError{
phase: validationPhaseReadmeParsing, phase: validationPhaseReadme,
errors: yamlParsingErrors, errors: yamlParsingErrors,
} }
} }
yamlValidationErrors := []error{} yamlValidationErrors := []error{}
for _, p := range profilesByNamespace { for _, p := range profilesByNamespace {
errors := validateContributorReadme(p) if errors := validateContributorReadme(p); len(errors) > 0 {
if len(errors) > 0 {
yamlValidationErrors = append(yamlValidationErrors, errors...) yamlValidationErrors = append(yamlValidationErrors, errors...)
continue continue
} }
} }
if len(yamlValidationErrors) != 0 { if len(yamlValidationErrors) != 0 {
return nil, validationPhaseError{ return nil, validationPhaseError{
phase: validationPhaseReadmeParsing, phase: validationPhaseReadme,
errors: yamlValidationErrors, errors: yamlValidationErrors,
} }
} }
@ -241,12 +236,13 @@ func aggregateContributorReadmeFiles() ([]readme, error) {
allReadmeFiles := []readme{} allReadmeFiles := []readme{}
errs := []error{} errs := []error{}
dirPath := ""
for _, e := range dirEntries { for _, e := range dirEntries {
dirPath := path.Join(rootRegistryPath, e.Name())
if !e.IsDir() { if !e.IsDir() {
continue continue
} }
dirPath = path.Join(rootRegistryPath, e.Name())
readmePath := path.Join(dirPath, "README.md") readmePath := path.Join(dirPath, "README.md")
rmBytes, err := os.ReadFile(readmePath) rmBytes, err := os.ReadFile(readmePath)
if err != nil { if err != nil {
@ -261,7 +257,7 @@ func aggregateContributorReadmeFiles() ([]readme, error) {
if len(errs) != 0 { if len(errs) != 0 {
return nil, validationPhaseError{ return nil, validationPhaseError{
phase: validationPhaseFileLoad, phase: validationPhaseFile,
errors: errs, errors: errs,
} }
} }
@ -269,18 +265,18 @@ func aggregateContributorReadmeFiles() ([]readme, error) {
return allReadmeFiles, nil return allReadmeFiles, nil
} }
func validateContributorRelativeUrls(contributors map[string]contributorProfileReadme) error { func validateContributorRelativeURLs(contributors map[string]contributorProfileReadme) error {
// This function only validates relative avatar URLs for now, but it can be // This function only validates relative avatar URLs for now, but it can be beefed up to validate more in the future.
// beefed up to validate more in the future.
var errs []error var errs []error
for _, con := range contributors { for _, con := range contributors {
// If the avatar URL is missing, we'll just assume that the Registry site build step will take care of filling
// in the data properly.
if con.frontmatter.AvatarURL == nil { if con.frontmatter.AvatarURL == nil {
continue continue
} }
isRelativeURL := strings.HasPrefix(*con.frontmatter.AvatarURL, ".") || if !strings.HasPrefix(*con.frontmatter.AvatarURL, ".") || !strings.HasPrefix(*con.frontmatter.AvatarURL, "/") {
strings.HasPrefix(*con.frontmatter.AvatarURL, "/")
if !isRelativeURL {
continue continue
} }
@ -291,10 +287,8 @@ func validateContributorRelativeUrls(contributors map[string]contributorProfileR
continue continue
} }
absolutePath := strings.TrimSuffix(con.filePath, "README.md") + absolutePath := strings.TrimSuffix(con.filePath, "README.md") + *con.frontmatter.AvatarURL
*con.frontmatter.AvatarURL if _, err := os.ReadFile(absolutePath); err != nil {
_, err := os.Stat(absolutePath)
if err != nil {
errs = append(errs, xerrors.Errorf("%q: relative avatar path %q does not point to image in file system", con.filePath, absolutePath)) errs = append(errs, xerrors.Errorf("%q: relative avatar path %q does not point to image in file system", con.filePath, absolutePath))
} }
} }
@ -303,7 +297,7 @@ func validateContributorRelativeUrls(contributors map[string]contributorProfileR
return nil return nil
} }
return validationPhaseError{ return validationPhaseError{
phase: validationPhaseAssetCrossReference, phase: validationPhaseCrossReference,
errors: errs, errors: errs,
} }
} }
@ -314,19 +308,18 @@ func validateAllContributorFiles() error {
return err return err
} }
log.Printf("Processing %d README files\n", len(allReadmeFiles)) logger.Info(context.Background(), "processing README files", "num_files", len(allReadmeFiles))
contributors, err := parseContributorFiles(allReadmeFiles) contributors, err := parseContributorFiles(allReadmeFiles)
if err != nil { if err != nil {
return err return err
} }
log.Printf("Processed %d README files as valid contributor profiles", len(contributors)) logger.Info(context.Background(), "processed README files as valid contributor profiles", "num_contributors", len(contributors))
err = validateContributorRelativeUrls(contributors) if err := validateContributorRelativeURLs(contributors); err != nil {
if err != nil {
return err return err
} }
log.Println("All relative URLs for READMEs are valid") logger.Info(context.Background(), "all relative URLs for READMEs are valid")
log.Printf("Processed all READMEs in the %q directory\n", rootRegistryPath) logger.Info(context.Background(), "processed all READMEs in directory", "dir", rootRegistryPath)
return nil return nil
} }

6
cmd/readmevalidation/errors.go
View File

@ -6,10 +6,8 @@ import (
"golang.org/x/xerrors" "golang.org/x/xerrors"
) )
// validationPhaseError represents an error that occurred during a specific // validationPhaseError represents an error that occurred during a specific phase of README validation. It should be
// phase of README validation. It should be used to collect ALL validation // used to collect ALL validation errors that happened during a specific phase, rather than the first one encountered.
// errors that happened during a specific phase, rather than the first one
// encountered.
type validationPhaseError struct { type validationPhaseError struct {
phase validationPhase phase validationPhase
errors []error errors []error

8
cmd/readmevalidation/main.go
View File

@ -7,7 +7,6 @@ package main
import ( import (
"context" "context"
"log"
"os" "os"
"cdr.dev/slog" "cdr.dev/slog"
@ -19,12 +18,11 @@ var logger = slog.Make(sloghuman.Sink(os.Stdout))
func main() { func main() {
logger.Info(context.Background(), "starting README validation") logger.Info(context.Background(), "starting README validation")
// If there are fundamental problems with how the repo is structured, we // If there are fundamental problems with how the repo is structured, we can't make any guarantees that any further
// can't make any guarantees that any further validations will be relevant // validations will be relevant or accurate.
// or accurate.
err := validateRepoStructure() err := validateRepoStructure()
if err != nil { if err != nil {
log.Println(err) logger.Error(context.Background(), "error when validating the repo structure", "error", err.Error())
os.Exit(1) os.Exit(1)
} }

122
cmd/readmevalidation/readmefiles.go
View File

@ -2,26 +2,55 @@ package main
import ( import (
"bufio" "bufio"
"fmt"
"regexp" "regexp"
"strings" "strings"
"golang.org/x/xerrors" "golang.org/x/xerrors"
) )
const rootRegistryPath = "./registry" // validationPhase represents a specific phase during README validation. It is expected that each phase is discrete, and
// errors during one will prevent a future phase from starting.
type validationPhase string
var supportedAvatarFileFormats = []string{".png", ".jpeg", ".jpg", ".gif", ".svg"} const (
rootRegistryPath = "./registry"
// readme represents a single README file within the repo (usually within the // --- validationPhases ---
// top-level "/registry" directory). // validationPhaseStructure indicates when the entire Registry
// directory is being verified for having all files be placed in the file
// system as expected.
validationPhaseStructure validationPhase = "File structure validation"
// ValidationPhaseFile indicates when README files are being read from
// the file system.
validationPhaseFile validationPhase = "Filesystem reading"
// ValidationPhaseReadme indicates when a README's frontmatter is
// being parsed as YAML. This phase does not include YAML validation.
validationPhaseReadme validationPhase = "README parsing"
// ValidationPhaseCrossReference indicates when a README's frontmatter
// is having all its relative URLs be validated for whether they point to
// valid resources.
validationPhaseCrossReference validationPhase = "Cross-referencing relative asset URLs"
// --- end of validationPhases ---.
)
var (
supportedAvatarFileFormats = []string{".png", ".jpeg", ".jpg", ".gif", ".svg"}
// Matches markdown headers, must be at the beginning of a line, such as "# " or "### ".
readmeHeaderRe = regexp.MustCompile(`^(#+)(\s*)`)
)
// readme represents a single README file within the repo (usually within the top-level "/registry" directory).
type readme struct { type readme struct {
filePath string filePath string
rawText string rawText string
} }
// separateFrontmatter attempts to separate a README file's frontmatter content // separateFrontmatter attempts to separate a README file's frontmatter content from the main README body, returning
// from the main README body, returning both values in that order. It does not // both values in that order. It does not validate whether the structure of the frontmatter is valid (i.e., that it's
// validate whether the structure of the frontmatter is valid (i.e., that it's
// structured as YAML). // structured as YAML).
func separateFrontmatter(readmeText string) (readmeFrontmatter string, readmeBody string, err error) { func separateFrontmatter(readmeText string) (readmeFrontmatter string, readmeBody string, err error) {
if readmeText == "" { if readmeText == "" {
@ -29,8 +58,9 @@ func separateFrontmatter(readmeText string) (readmeFrontmatter string, readmeBod
} }
const fence = "---" const fence = "---"
fm := ""
body := "" var fm strings.Builder
var body strings.Builder
fenceCount := 0 fenceCount := 0
lineScanner := bufio.NewScanner(strings.NewReader(strings.TrimSpace(readmeText))) lineScanner := bufio.NewScanner(strings.NewReader(strings.TrimSpace(readmeText)))
@ -40,36 +70,32 @@ func separateFrontmatter(readmeText string) (readmeFrontmatter string, readmeBod
fenceCount++ fenceCount++
continue continue
} }
// Break early if the very first line wasn't a fence, because then we // Break early if the very first line wasn't a fence, because then we know for certain that the README has problems.
// know for certain that the README has problems.
if fenceCount == 0 { if fenceCount == 0 {
break break
} }
// It should be safe to trim each line of the frontmatter on a per-line // It should be safe to trim each line of the frontmatter on a per-line basis, because there shouldn't be any
// basis, because there shouldn't be any extra meaning attached to the // extra meaning attached to the indentation. The same does NOT apply to the README; best we can do is gather
// indentation. The same does NOT apply to the README; best we can do is // all the lines and then trim around it.
// gather all the lines, and then trim around it.
if inReadmeBody := fenceCount >= 2; inReadmeBody { if inReadmeBody := fenceCount >= 2; inReadmeBody {
body += nextLine + "\n" fmt.Fprintf(&body, "%s\n", nextLine)
} else { } else {
fm += strings.TrimSpace(nextLine) + "\n" fmt.Fprintf(&fm, "%s\n", strings.TrimSpace(nextLine))
} }
} }
if fenceCount < 2 { if fenceCount < 2 {
return "", "", xerrors.New("README does not have two sets of frontmatter fences") return "", "", xerrors.New("README does not have two sets of frontmatter fences")
} }
if fm == "" { if fm.Len() == 0 {
return "", "", xerrors.New("readme has frontmatter fences but no frontmatter content") return "", "", xerrors.New("readme has frontmatter fences but no frontmatter content")
} }
return fm, strings.TrimSpace(body), nil return fm.String(), strings.TrimSpace(body.String()), nil
} }
var readmeHeaderRe = regexp.MustCompile(`^(#+)(\s*)`) // TODO: This seems to work okay for now, but the really proper way of doing this is by parsing this as an AST, and then
// checking the resulting nodes.
// Todo: This seems to work okay for now, but the really proper way of doing
// this is by parsing this as an AST, and then checking the resulting nodes.
func validateReadmeBody(body string) []error { func validateReadmeBody(body string) []error {
trimmed := strings.TrimSpace(body) trimmed := strings.TrimSpace(body)
@ -77,9 +103,8 @@ func validateReadmeBody(body string) []error {
return []error{xerrors.New("README body is empty")} return []error{xerrors.New("README body is empty")}
} }
// If the very first line of the README, there's a risk that the rest of the // If the very first line of the README doesn't start with an ATX-style H1 header, there's a risk that the rest of the
// validation logic will break, since we don't have many guarantees about // validation logic will break, since we don't have many guarantees about how the README is actually structured.
// how the README is actually structured.
if !strings.HasPrefix(trimmed, "# ") { if !strings.HasPrefix(trimmed, "# ") {
return []error{xerrors.New("README body must start with ATX-style h1 header (i.e., \"# \")")} return []error{xerrors.New("README body must start with ATX-style h1 header (i.e., \"# \")")}
} }
@ -93,9 +118,8 @@ func validateReadmeBody(body string) []error {
for lineScanner.Scan() { for lineScanner.Scan() {
nextLine := lineScanner.Text() nextLine := lineScanner.Text()
// Have to check this because a lot of programming languages support # // Have to check this because a lot of programming languages support # comments (including Terraform), and
// comments (including Terraform), and without any context, there's no // without any context, there's no way to tell the difference between a markdown header and code comment.
// way to tell the difference between a markdown header and code comment.
if strings.HasPrefix(nextLine, "```") { if strings.HasPrefix(nextLine, "```") {
isInCodeBlock = !isInCodeBlock isInCodeBlock = !isInCodeBlock
continue continue
@ -109,8 +133,8 @@ func validateReadmeBody(body string) []error {
continue continue
} }
spaceAfterHeader := headerGroups[2] // In the Markdown spec it is mandatory to have a space following the header # symbol(s).
if spaceAfterHeader == "" { if headerGroups[2] == "" {
errs = append(errs, xerrors.New("header does not have space between header characters and main header text")) errs = append(errs, xerrors.New("header does not have space between header characters and main header text"))
} }
@ -121,8 +145,7 @@ func validateReadmeBody(body string) []error {
continue continue
} }
// If we have obviously invalid headers, it's not really safe to keep // If we have obviously invalid headers, it's not really safe to keep proceeding with the rest of the content.
// proceeding with the rest of the content.
if nextHeaderLevel == 1 { if nextHeaderLevel == 1 {
errs = append(errs, xerrors.New("READMEs cannot contain more than h1 header")) errs = append(errs, xerrors.New("READMEs cannot contain more than h1 header"))
break break
@ -132,43 +155,16 @@ func validateReadmeBody(body string) []error {
break break
} }
// This is something we need to enforce for accessibility, not just for // This is something we need to enforce for accessibility, not just for the Registry website, but also when
// the Registry website, but also when users are viewing the README // users are viewing the README files in the GitHub web view.
// files in the GitHub web view.
if nextHeaderLevel > latestHeaderLevel && nextHeaderLevel != (latestHeaderLevel+1) { if nextHeaderLevel > latestHeaderLevel && nextHeaderLevel != (latestHeaderLevel+1) {
errs = append(errs, xerrors.New("headers are not allowed to increase more than 1 level at a time")) errs = append(errs, xerrors.New("headers are not allowed to increase more than 1 level at a time"))
continue continue
} }
// As long as the above condition passes, there's no problems with // As long as the above condition passes, there's no problems with going up a header level or going down 1+ header levels.
// going up a header level or going down 1+ header levels.
latestHeaderLevel = nextHeaderLevel latestHeaderLevel = nextHeaderLevel
} }
return errs return errs
} }
// validationPhase represents a specific phase during README validation. It is
// expected that each phase is discrete, and errors during one will prevent a
// future phase from starting.
type validationPhase string
const (
// ValidationPhaseFileStructureValidation indicates when the entire Registry
// directory is being verified for having all files be placed in the file
// system as expected.
validationPhaseFileStructureValidation validationPhase = "File structure validation"
// ValidationPhaseFileLoad indicates when README files are being read from
// the file system.
validationPhaseFileLoad = "Filesystem reading"
// ValidationPhaseReadmeParsing indicates when a README's frontmatter is
// being parsed as YAML. This phase does not include YAML validation.
validationPhaseReadmeParsing = "README parsing"
// ValidationPhaseAssetCrossReference indicates when a README's frontmatter
// is having all its relative URLs be validated for whether they point to
// valid resources.
validationPhaseAssetCrossReference = "Cross-referencing relative asset URLs"
)

55
cmd/readmevalidation/repostructure.go
View File

@ -13,40 +13,33 @@ import (
var supportedUserNameSpaceDirectories = append(supportedResourceTypes, ".icons", ".images") var supportedUserNameSpaceDirectories = append(supportedResourceTypes, ".icons", ".images")
func validateCoderResourceSubdirectory(dirPath string) []error { func validateCoderResourceSubdirectory(dirPath string) []error {
errs := []error{}
subDir, err := os.Stat(dirPath) subDir, err := os.Stat(dirPath)
if err != nil { if err != nil {
// It's valid for a specific resource directory not to exist. It's just // It's valid for a specific resource directory not to exist. It's just that if it does exist, it must follow specific rules.
// that if it does exist, it must follow specific rules.
if !errors.Is(err, os.ErrNotExist) { if !errors.Is(err, os.ErrNotExist) {
errs = append(errs, addFilePathToError(dirPath, err)) return []error{addFilePathToError(dirPath, err)}
} }
return errs
} }
if !subDir.IsDir() { if !subDir.IsDir() {
errs = append(errs, xerrors.Errorf("%q: path is not a directory", dirPath)) return []error{xerrors.Errorf("%q: path is not a directory", dirPath)}
return errs
} }
files, err := os.ReadDir(dirPath) files, err := os.ReadDir(dirPath)
if err != nil { if err != nil {
errs = append(errs, addFilePathToError(dirPath, err)) return []error{addFilePathToError(dirPath, err)}
return errs
} }
errs := []error{}
for _, f := range files { for _, f := range files {
// The .coder subdirectories are sometimes generated as part of Bun // The .coder subdirectories are sometimes generated as part of Bun tests. These subdirectories will never be
// tests. These subdirectories will never be committed to the repo, but // committed to the repo, but in the off chance that they don't get cleaned up properly, we want to skip over them.
// in the off chance that they don't get cleaned up properly, we want to
// skip over them.
if !f.IsDir() || f.Name() == ".coder" { if !f.IsDir() || f.Name() == ".coder" {
continue continue
} }
resourceReadmePath := path.Join(dirPath, f.Name(), "README.md") resourceReadmePath := path.Join(dirPath, f.Name(), "README.md")
_, err := os.Stat(resourceReadmePath) if _, err := os.Stat(resourceReadmePath); err != nil {
if err != nil {
if errors.Is(err, os.ErrNotExist) { if errors.Is(err, os.ErrNotExist) {
errs = append(errs, xerrors.Errorf("%q: 'README.md' does not exist", resourceReadmePath)) errs = append(errs, xerrors.Errorf("%q: 'README.md' does not exist", resourceReadmePath))
} else { } else {
@ -55,8 +48,7 @@ func validateCoderResourceSubdirectory(dirPath string) []error {
} }
mainTerraformPath := path.Join(dirPath, f.Name(), "main.tf") mainTerraformPath := path.Join(dirPath, f.Name(), "main.tf")
_, err = os.Stat(mainTerraformPath) if _, err := os.Stat(mainTerraformPath); err != nil {
if err != nil {
if errors.Is(err, os.ErrNotExist) { if errors.Is(err, os.ErrNotExist) {
errs = append(errs, xerrors.Errorf("%q: 'main.tf' file does not exist", mainTerraformPath)) errs = append(errs, xerrors.Errorf("%q: 'main.tf' file does not exist", mainTerraformPath))
} else { } else {
@ -64,7 +56,6 @@ func validateCoderResourceSubdirectory(dirPath string) []error {
} }
} }
} }
return errs return errs
} }
@ -83,8 +74,7 @@ func validateRegistryDirectory() []error {
} }
contributorReadmePath := path.Join(dirPath, "README.md") contributorReadmePath := path.Join(dirPath, "README.md")
_, err := os.Stat(contributorReadmePath) if _, err := os.Stat(contributorReadmePath); err != nil {
if err != nil {
allErrs = append(allErrs, err) allErrs = append(allErrs, err)
} }
@ -95,8 +85,7 @@ func validateRegistryDirectory() []error {
} }
for _, f := range files { for _, f := range files {
// Todo: Decide if there's anything more formal that we want to // TODO: Decide if there's anything more formal that we want to ensure about non-directories scoped to user namespaces.
// ensure about non-directories scoped to user namespaces.
if !f.IsDir() { if !f.IsDir() {
continue continue
} }
@ -110,8 +99,7 @@ func validateRegistryDirectory() []error {
} }
if slices.Contains(supportedResourceTypes, segment) { if slices.Contains(supportedResourceTypes, segment) {
errs := validateCoderResourceSubdirectory(filePath) if errs := validateCoderResourceSubdirectory(filePath); len(errs) != 0 {
if len(errs) != 0 {
allErrs = append(allErrs, errs...) allErrs = append(allErrs, errs...)
} }
} }
@ -122,20 +110,19 @@ func validateRegistryDirectory() []error {
} }
func validateRepoStructure() error { func validateRepoStructure() error {
var problems []error var errs []error
if errs := validateRegistryDirectory(); len(errs) != 0 { if vrdErrs := validateRegistryDirectory(); len(vrdErrs) != 0 {
problems = append(problems, errs...) errs = append(errs, vrdErrs...)
} }
_, err := os.Stat("./.icons") if _, err := os.Stat("./.icons"); err != nil {
if err != nil { errs = append(errs, xerrors.New("missing top-level .icons directory (used for storing reusable Coder resource icons)"))
problems = append(problems, xerrors.New("missing top-level .icons directory (used for storing reusable Coder resource icons)"))
} }
if len(problems) != 0 { if len(errs) != 0 {
return validationPhaseError{ return validationPhaseError{
phase: validationPhaseFileStructureValidation, phase: validationPhaseStructure,
errors: problems, errors: errs,
} }
} }
return nil return nil

6
go.mod
View File

@ -20,7 +20,7 @@ require (
github.com/rivo/uniseg v0.4.4 // indirect github.com/rivo/uniseg v0.4.4 // indirect
go.opentelemetry.io/otel v1.16.0 // indirect go.opentelemetry.io/otel v1.16.0 // indirect
go.opentelemetry.io/otel/trace v1.16.0 // indirect go.opentelemetry.io/otel/trace v1.16.0 // indirect
golang.org/x/crypto v0.35.0 // indirect golang.org/x/crypto v0.11.0 // indirect
golang.org/x/sys v0.30.0 // indirect golang.org/x/sys v0.10.0 // indirect
golang.org/x/term v0.29.0 // indirect golang.org/x/term v0.10.0 // indirect
) )

20
go.sum
View File

@ -51,17 +51,17 @@ go.opentelemetry.io/otel/sdk v1.16.0 h1:Z1Ok1YsijYL0CSJpHt4cS3wDDh7p572grzNrBMiM
go.opentelemetry.io/otel/sdk v1.16.0/go.mod h1:tMsIuKXuuIWPBAOrH+eHtvhTL+SntFtXF9QD68aP6p4= go.opentelemetry.io/otel/sdk v1.16.0/go.mod h1:tMsIuKXuuIWPBAOrH+eHtvhTL+SntFtXF9QD68aP6p4=
go.opentelemetry.io/otel/trace v1.16.0 h1:8JRpaObFoW0pxuVPapkgH8UhHQj+bJW8jJsCZEu5MQs= go.opentelemetry.io/otel/trace v1.16.0 h1:8JRpaObFoW0pxuVPapkgH8UhHQj+bJW8jJsCZEu5MQs=
go.opentelemetry.io/otel/trace v1.16.0/go.mod h1:Yt9vYq1SdNz3xdjZZK7wcXv1qv2pwLkqr2QVwea0ef0= go.opentelemetry.io/otel/trace v1.16.0/go.mod h1:Yt9vYq1SdNz3xdjZZK7wcXv1qv2pwLkqr2QVwea0ef0=
golang.org/x/crypto v0.35.0 h1:b15kiHdrGCHrP6LvwaQ3c03kgNhhiMgvlhxHQhmg2Xs= golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA=
golang.org/x/crypto v0.35.0/go.mod h1:dy7dXNW32cAb/6/PRuTNsix8T+vJAqvuIy5Bli/x0YQ= golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio=
golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50=
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA=
golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.29.0 h1:L6pJp37ocefwRRtYPKSWOWzOtWSxVajvz2ldH/xi3iU= golang.org/x/term v0.10.0 h1:3R7pNqamzBraeqj/Tj8qt1aQ2HpmlC+Cx/qL/7hn4/c=
golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s= golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o=
golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4=
golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY= golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY=
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90=
google.golang.org/genproto v0.0.0-20230726155614-23370e0ffb3e h1:xIXmWJ303kJCuogpj0bHq+dcjcZHU+XFyc1I0Yl9cRg= google.golang.org/genproto v0.0.0-20230726155614-23370e0ffb3e h1:xIXmWJ303kJCuogpj0bHq+dcjcZHU+XFyc1I0Yl9cRg=