Add URL support and YAML export

go.mod

@@ -2,6 +2,17 @@ module github.com/JordanCoin/docmap
 
 go 1.25.5
 
-require github.com/ledongthuc/pdf v0.0.0-20250511090121-5959a4027728
+require (
+	github.com/klippa-app/go-pdfium v1.17.3
+	github.com/ledongthuc/pdf v0.0.0-20250511090121-5959a4027728
+	gopkg.in/yaml.v3 v3.0.1
+)
 
-require gopkg.in/yaml.v3 v3.0.1 // indirect
+require (
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/jolestar/go-commons-pool/v2 v2.1.2 // indirect
+	github.com/tetratelabs/wazero v1.11.0 // indirect
+	golang.org/x/net v0.50.0 // indirect
+	golang.org/x/sys v0.41.0 // indirect
+	golang.org/x/text v0.34.0 // indirect
+)

go.sum

@@ -1,5 +1,54 @@
+github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0=
+github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
+github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
+github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
+github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI=
+github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8=
+github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
+github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
+github.com/google/pprof v0.0.0-20260115054156-294ebfa9ad83 h1:z2ogiKUYzX5Is6zr/vP9vJGqPwcdqsWjOt+V8J7+bTc=
+github.com/google/pprof v0.0.0-20260115054156-294ebfa9ad83/go.mod h1:MxpfABSjhmINe3F1It9d+8exIHFvUqtLIRCdOGNXqiI=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/jolestar/go-commons-pool/v2 v2.1.2 h1:E+XGo58F23t7HtZiC/W6jzO2Ux2IccSH/yx4nD+J1CM=
+github.com/jolestar/go-commons-pool/v2 v2.1.2/go.mod h1:r4NYccrkS5UqP1YQI1COyTZ9UjPJAAGTUxzcsK1kqhY=
+github.com/klippa-app/go-pdfium v1.17.3 h1:j+3VnnJvnVdLV16fPugN43GvucyfXIDXSg0Z7wSQ0yg=
+github.com/klippa-app/go-pdfium v1.17.3/go.mod h1:T7ZFRT9CpW8TKG+P5/4cNa/OvTzSZ+CqzasPz5UeuV4=
 github.com/ledongthuc/pdf v0.0.0-20250511090121-5959a4027728 h1:QwWKgMY28TAXaDl+ExRDqGQltzXqN/xypdKP86niVn8=
 github.com/ledongthuc/pdf v0.0.0-20250511090121-5959a4027728/go.mod h1:1fEHWurg7pvf5SG6XNE5Q8UZmOwex51Mkx3SLhrW5B4=
+github.com/onsi/ginkgo/v2 v2.28.1 h1:S4hj+HbZp40fNKuLUQOYLDgZLwNUVn19N3Atb98NCyI=
+github.com/onsi/ginkgo/v2 v2.28.1/go.mod h1:CLtbVInNckU3/+gC8LzkGUb9oF+e8W8TdUsxPwvdOgE=
+github.com/onsi/gomega v1.39.1 h1:1IJLAad4zjPn2PsnhH70V4DKRFlrCzGBNrNaru+Vf28=
+github.com/onsi/gomega v1.39.1/go.mod h1:hL6yVALoTOxeWudERyfppUcZXjMwIMLnuSfruD2lcfg=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
+github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
+github.com/tetratelabs/wazero v1.11.0 h1:+gKemEuKCTevU4d7ZTzlsvgd1uaToIDtlQlmNbwqYhA=
+github.com/tetratelabs/wazero v1.11.0/go.mod h1:eV28rsN8Q+xwjogd7f4/Pp4xFxO7uOGbLcD/LzB1wiU=
+go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
+go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
+golang.org/x/mod v0.32.0 h1:9F4d3PHLljb6x//jOyokMv3eX+YDeepZSEo3mFJy93c=
+golang.org/x/mod v0.32.0/go.mod h1:SgipZ/3h2Ci89DlEtEXWUk/HteuRin+HHhN+WbNhguU=
+golang.org/x/net v0.50.0 h1:ucWh9eiCGyDR3vtzso0WMQinm2Dnt8cFMuQa9K33J60=
+golang.org/x/net v0.50.0/go.mod h1:UgoSli3F/pBgdJBHCTc+tp3gmrU4XswgGRgtnwWTfyM=
+golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
+golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
+golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k=
+golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
+golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk=
+golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA=
+golang.org/x/tools v0.41.0 h1:a9b8iMweWG+S0OBnlU36rzLp20z1Rp10w+IY2czHTQc=
+golang.org/x/tools v0.41.0/go.mod h1:XSY6eDqxVNiYgezAVqqCeihT4j1U2CCsqvH3WhQpnlg=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

main.go

@@ -66,6 +66,7 @@ func main() {
 	var sectionFilter string
 	var expandSection string
 	var searchQuery string
+	var outputFile string
 	var showRefs bool
 	var jsonMode bool
 	for i := 2; i < len(os.Args); i++ {
@@ -85,13 +86,61 @@ func main() {
 				searchQuery = os.Args[i+1]
 				i++
 			}
+		case "--output", "-o":
+			if i+1 < len(os.Args) {
+				outputFile = os.Args[i+1]
+				i++
+			}
 		case "--refs", "-r":
 			showRefs = true
 		case "--json", "-j":
 			jsonMode = true
 		}
 	}
 
+	// Check if target is a URL
+	isURL := strings.HasPrefix(target, "http://") || strings.HasPrefix(target, "https://")
+	if isURL {
+		doc, err := parser.ParseURL(target)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+			os.Exit(1)
+		}
+
+		// Use the URL's last path segment as filename
+		parts := strings.Split(strings.TrimRight(target, "/"), "/")
+		doc.Filename = parts[len(parts)-1]
+		if doc.Filename == "" {
+			doc.Filename = target
+		}
+
+		if outputFile != "" {
+			yamlContent, err := parser.ExportYAML(doc)
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "Error exporting YAML: %v\n", err)
+				os.Exit(1)
+			}
+			if err := os.WriteFile(outputFile, []byte(yamlContent), 0644); err != nil {
+				fmt.Fprintf(os.Stderr, "Error writing file: %v\n", err)
+				os.Exit(1)
+			}
+			fmt.Fprintf(os.Stderr, "Saved to %s\n", outputFile)
+		}
+
+		if jsonMode {
+			outputJSON([]*parser.Document{doc}, target)
+		} else if searchQuery != "" {
+			render.SearchResults([]*parser.Document{doc}, searchQuery)
+		} else if expandSection != "" {
+			render.ExpandSection(doc, expandSection)
+		} else if sectionFilter != "" {
+			render.FilteredTree(doc, sectionFilter)
+		} else if outputFile == "" {
+			render.Tree(doc)
+		}
+		return
+	}
+
 	// Check if target is a directory
 	info, err := os.Stat(target)
 	if err != nil {
@@ -154,6 +203,19 @@ func main() {
 		parts := strings.Split(target, "/")
 		doc.Filename = parts[len(parts)-1]
 
+		if outputFile != "" {
+			yamlContent, err := parser.ExportYAML(doc)
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "Error exporting YAML: %v\n", err)
+				os.Exit(1)
+			}
+			if err := os.WriteFile(outputFile, []byte(yamlContent), 0644); err != nil {
+				fmt.Fprintf(os.Stderr, "Error writing file: %v\n", err)
+				os.Exit(1)
+			}
+			fmt.Fprintf(os.Stderr, "Saved to %s\n", outputFile)
+		}
+
 		if jsonMode {
 			absPath, _ := filepath.Abs(target)
 			outputJSON([]*parser.Document{doc}, absPath)
@@ -163,7 +225,7 @@ func main() {
 			render.ExpandSection(doc, expandSection)
 		} else if sectionFilter != "" {
 			render.FilteredTree(doc, sectionFilter)
-		} else {
+		} else if outputFile == "" {
 			render.Tree(doc)
 		}
 	}
@@ -281,7 +343,7 @@ func printUsage() {
 	fmt.Println(`docmap - instant documentation structure for LLMs and humans
 
 Usage:
-  docmap <file.md|file.pdf|file.yaml|dir> [flags]
+  docmap <file.md|file.pdf|file.yaml|url|dir> [flags]
 
 Examples:
   docmap .                          # All markdown, PDF, and YAML files
@@ -292,12 +354,19 @@ Examples:
   docmap README.md --section "API"  # Filter to section
   docmap README.md --expand "API"   # Show section content
   docmap . --refs                   # Show cross-references between docs
-  docmap docs/ --search "auth"     # Search across all files
+  docmap docs/ --search "auth"      # Search across all files
+
+URL Support:
+  docmap https://example.com/docs                     # Map a web page
+  docmap https://example.com/docs --search "auth"     # Search sections
+  docmap https://example.com/docs -o docs.yaml        # Save as YAML
+  docmap docs.yaml --search "auth"                    # Fast local access
 
 Flags:
   --search <query>       Search sections across all files
   -s, --section <name>   Filter to a specific section
   -e, --expand <name>    Show full content of a section
+  -o, --output <file>    Export structure as YAML file
   -r, --refs             Show cross-references between markdown files
   -j, --json             Output JSON format
   -v, --version          Print version
@@ -311,5 +380,9 @@ YAML Support:
   Maps keys to sections with nested children. Sequences use name/id/title
   fields for titles when available, falling back to key: value or [N].
 
+URL Support:
+  Uses headless Chrome to render web pages, then extracts heading structure
+  from font sizes. Requires Chrome/Chromium installed, or set CHROME_PATH.
+
 More info: https://github.com/JordanCoin/docmap`)
 }

parser/export.go

@@ -0,0 +1,57 @@
+package parser
+
+import (
+	"fmt"
+
+	"gopkg.in/yaml.v3"
+)
+
+// yamlSection is the serializable form of a Section for YAML export.
+type yamlSection struct {
+	Title    string        `yaml:"title"`
+	Content  string        `yaml:"content,omitempty"`
+	Tokens   int           `yaml:"tokens"`
+	Children []yamlSection `yaml:"children,omitempty"`
+}
+
+// yamlDocument is the serializable form of a Document for YAML export.
+type yamlDocument struct {
+	Docmap   string        `yaml:"docmap"`
+	Filename string        `yaml:"filename,omitempty"`
+	Tokens   int           `yaml:"tokens"`
+	Sections []yamlSection `yaml:"sections"`
+}
+
+// ExportYAML serializes a Document to structured YAML.
+// The output can be read back by ParseYAML to reconstruct the document.
+func ExportYAML(doc *Document) (string, error) {
+	yd := yamlDocument{
+		Docmap:   "1.0",
+		Filename: doc.Filename,
+		Tokens:   doc.TotalTokens,
+		Sections: convertToYAMLSections(doc.Sections),
+	}
+
+	data, err := yaml.Marshal(yd)
+	if err != nil {
+		return "", fmt.Errorf("failed to marshal YAML: %w", err)
+	}
+
+	return string(data), nil
+}
+
+func convertToYAMLSections(sections []*Section) []yamlSection {
+	var result []yamlSection
+	for _, s := range sections {
+		ys := yamlSection{
+			Title:    s.Title,
+			Tokens:   s.Tokens,
+			Children: convertToYAMLSections(s.Children),
+		}
+		if s.Content != "" {
+			ys.Content = s.Content
+		}
+		result = append(result, ys)
+	}
+	return result
+}