main.go

// Command conformance runs each .http file in a directory through both
// httpsuite and the JetBrains HTTP Client CLI (ijhttp), then diffs their JUnit
// reports by test name. It is the empirical "does httpsuite match JetBrains?"
// check.
//
// Usage:
//
//	go run ./conformance [flags]
//
// If ijhttp is not found on PATH (it requires a JDK), the harness still runs the
// httpsuite side and prints its results, then reports that the comparison was
// skipped. Point --ijhttp at a binary or a wrapper script (e.g. a `docker run`
// shim using the JetBrains image) to enable the side-by-side diff.
package main

import (
	"bytes"
	"encoding/xml"
	"flag"
	"fmt"
	"io"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"sort"
	"strings"
	"time"
)

func main() {
	os.Exit(run())
}

func run() int {
	var (
		dir       = flag.String("dir", "conformance/tests", "directory of .http conformance files")
		ijhttp    = flag.String("ijhttp", lookIjhttp(), "path to the ijhttp binary (or a wrapper); empty to skip")
		addr      = flag.String("addr", "127.0.0.1:8799", "address for the example server (must match @base in the files)")
		httpsuite = flag.String("httpsuite", "", "path to a prebuilt httpsuite binary (default: build from source)")
		keep      = flag.Bool("keep", false, "keep the temporary work directory")
	)
	flag.Parse()

	work, err := os.MkdirTemp("", "httpsuite-conformance-")
	if err != nil {
		fmt.Fprintln(os.Stderr, "mktemp:", err)
		return 2
	}
	if *keep {
		fmt.Println("work dir:", work)
	} else {
		defer os.RemoveAll(work)
	}

	// Build httpsuite unless a binary was supplied.
	hsBin := *httpsuite
	if hsBin == "" {
		hsBin = filepath.Join(work, "httpsuite")
		if err := goBuild(hsBin, "."); err != nil {
			fmt.Fprintln(os.Stderr, "building httpsuite:", err)
			return 2
		}
	}

	// Build and start the example server.
	srvBin := filepath.Join(work, "server")
	if err := goBuild(srvBin, "./example/server"); err != nil {
		fmt.Fprintln(os.Stderr, "building example server:", err)
		return 2
	}
	srv := exec.Command(srvBin, "--addr", *addr)
	srv.Stdout, srv.Stderr = io.Discard, io.Discard
	if err := srv.Start(); err != nil {
		fmt.Fprintln(os.Stderr, "starting server:", err)
		return 2
	}
	defer func() { _ = srv.Process.Kill() }()
	if err := waitReady("http://"+*addr+"/users", 5*time.Second); err != nil {
		fmt.Fprintln(os.Stderr, "server not ready:", err)
		return 2
	}

	files, err := filepath.Glob(filepath.Join(*dir, "*.http"))
	if err != nil || len(files) == 0 {
		fmt.Fprintf(os.Stderr, "no .http files in %s\n", *dir)
		return 2
	}
	sort.Strings(files)

	haveIjhttp := *ijhttp != ""
	if !haveIjhttp {
		fmt.Println("NOTE: ijhttp not found — running httpsuite only (comparison skipped).")
		fmt.Println("      Install ijhttp (brew install ijhttp) or pass --ijhttp <path|wrapper> to compare.")
		fmt.Println()
	}

	divergences := 0
	failedRuns := 0
	for _, f := range files {
		fmt.Printf("── %s\n", filepath.Base(f))

		hs, err := runHTTPSuite(hsBin, f, work)
		if err != nil {
			fmt.Printf("   httpsuite: FAILED TO RUN: %v\n\n", err)
			failedRuns++
			continue
		}

		if !haveIjhttp {
			for _, name := range sortedKeys(hs) {
				fmt.Printf("   %-40s httpsuite=%s\n", name, passStr(hs[name]))
			}
			fmt.Println()
			continue
		}

		ij, err := runIjhttp(*ijhttp, f, work)
		if err != nil {
			fmt.Printf("   ijhttp: FAILED TO RUN: %v\n\n", err)
			failedRuns++
			continue
		}

		d := compare(hs, ij)
		divergences += d
		fmt.Println()
	}

	fmt.Println(strings.Repeat("─", 60))
	if !haveIjhttp {
		fmt.Println("Comparison skipped (no ijhttp). httpsuite results shown above.")
		if failedRuns > 0 {
			fmt.Printf("%d file(s) failed to run.\n", failedRuns)
			return 1
		}
		return 0
	}
	if failedRuns > 0 {
		fmt.Printf("%d file(s) failed to run.\n", failedRuns)
	}
	if divergences == 0 {
		fmt.Println("CONFORMANT: httpsuite and ijhttp agree on every test.")
		if failedRuns > 0 {
			return 1
		}
		return 0
	}
	fmt.Printf("DIVERGENT: %d test(s) differ between httpsuite and ijhttp.\n", divergences)
	return 1
}

// compare prints per-test agreement and returns the number of divergences.
func compare(hs, ij map[string]bool) int {
	names := map[string]bool{}
	for n := range hs {
		names[n] = true
	}
	for n := range ij {
		names[n] = true
	}
	div := 0
	for _, name := range sortedKeys(names) {
		hv, hok := hs[name]
		iv, iok := ij[name]
		switch {
		case hok && iok && hv == iv:
			fmt.Printf("   ✓ %-40s httpsuite=%s ijhttp=%s\n", name, passStr(hv), passStr(iv))
		case hok && iok:
			fmt.Printf("   ✗ %-40s httpsuite=%s ijhttp=%s  <-- MISMATCH\n", name, passStr(hv), passStr(iv))
			div++
		case hok && !iok:
			fmt.Printf("   ! %-40s httpsuite=%s ijhttp=(absent)  <-- HTTPSUITE ONLY\n", name, passStr(hv))
			div++
		default:
			fmt.Printf("   ! %-40s httpsuite=(absent) ijhttp=%s  <-- IJHTTP ONLY\n", name, passStr(iv))
			div++
		}
	}
	return div
}

func runHTTPSuite(bin, file, work string) (map[string]bool, error) {
	report := filepath.Join(work, "hs-"+safe(file)+".xml")
	cmd := exec.Command(bin, "--report", report, file)
	cmd.Stdout, cmd.Stderr = io.Discard, io.Discard
	_ = cmd.Run() // non-zero exit just means some tests failed; the report is what matters
	data, err := os.ReadFile(report)
	if err != nil {
		return nil, fmt.Errorf("no report produced: %w", err)
	}
	return parseJUnit(data)
}

// ijhttpPseudoTests are lifecycle testcases ijhttp emits per request that have
// no client.test equivalent in httpsuite; they are filtered from comparison.
var ijhttpPseudoTests = map[string]bool{
	"Response": true, "Response Handler": true, "Request": true, "Pre-request Handler": true,
}

func runIjhttp(bin, file, work string) (map[string]bool, error) {
	abs, _ := filepath.Abs(file)
	out := filepath.Join(work, "ij-"+safe(file))
	if err := os.MkdirAll(out, 0o755); err != nil {
		return nil, err
	}
	// ijhttp writes report.xml into the directory given to --report=<dir>.
	cmd := exec.Command(bin, "--report="+out, "--no-progress", abs)
	cmd.Stdout, cmd.Stderr = io.Discard, io.Discard
	_ = cmd.Run()
	xmls, _ := filepath.Glob(filepath.Join(out, "*.xml"))
	if len(xmls) == 0 {
		xmls, _ = filepath.Glob(filepath.Join(out, "*", "*.xml"))
	}
	if len(xmls) == 0 {
		return nil, fmt.Errorf("no report produced (is ijhttp working?)")
	}
	data, err := os.ReadFile(xmls[0])
	if err != nil {
		return nil, err
	}
	m, err := parseJUnit(data)
	if err != nil {
		return nil, err
	}
	for name := range ijhttpPseudoTests {
		delete(m, name)
	}
	return m, nil
}

// parseJUnit collects every <testcase> in a JUnit document (at any nesting) into
// a name->passed map. A testcase is failed if it contains a <failure>/<error>.
// Duplicate names collapse to a pass only if every instance passed.
func parseJUnit(data []byte) (map[string]bool, error) {
	dec := xml.NewDecoder(bytes.NewReader(data))
	results := map[string]bool{}
	inCase := false
	failed := false
	name := ""
	for {
		tok, err := dec.Token()
		if err == io.EOF {
			break
		}
		if err != nil {
			return nil, err
		}
		switch t := tok.(type) {
		case xml.StartElement:
			switch t.Name.Local {
			case "testcase":
				inCase, failed, name = true, false, attrOf(t, "name")
			case "failure", "error":
				if inCase {
					failed = true
				}
			}
		case xml.EndElement:
			if t.Name.Local == "testcase" && inCase {
				if prev, ok := results[name]; ok {
					results[name] = prev && !failed
				} else {
					results[name] = !failed
				}
				inCase = false
			}
		}
	}
	return results, nil
}

func attrOf(e xml.StartElement, key string) string {
	for _, a := range e.Attr {
		if a.Name.Local == key {
			return a.Value
		}
	}
	return ""
}

func goBuild(out, pkg string) error {
	cmd := exec.Command("go", "build", "-o", out, pkg)
	var stderr bytes.Buffer
	cmd.Stderr = &stderr
	if err := cmd.Run(); err != nil {
		return fmt.Errorf("%v: %s", err, strings.TrimSpace(stderr.String()))
	}
	return nil
}

func waitReady(url string, timeout time.Duration) error {
	deadline := time.Now().Add(timeout)
	for time.Now().Before(deadline) {
		resp, err := http.Get(url)
		if err == nil {
			resp.Body.Close()
			return nil
		}
		time.Sleep(50 * time.Millisecond)
	}
	return fmt.Errorf("timed out waiting for %s", url)
}

func lookIjhttp() string {
	if p, err := exec.LookPath("ijhttp"); err == nil {
		return p
	}
	return ""
}

func sortedKeys(m map[string]bool) []string {
	out := make([]string, 0, len(m))
	for k := range m {
		out = append(out, k)
	}
	sort.Strings(out)
	return out
}

func passStr(b bool) string {
	if b {
		return "PASS"
	}
	return "FAIL"
}

func safe(path string) string {
	return strings.NewReplacer("/", "_", "\\", "_", ".", "_").Replace(filepath.Base(path))
}