From 558cbe8962e8f4cd013b7d266434b0539b42d1e4 Mon Sep 17 00:00:00 2001 From: Zhao Chen Date: Thu, 26 Mar 2026 18:32:39 +0800 Subject: [PATCH 1/6] feat: migrate PathFilter to doublestar, add include pattern support - Migrate exclude matching from filepath.Match to doublestar.PathMatch for consistency with pkg/backend/fetch.go - Rename patterns field to excludePatterns, add includePatterns field - Add MatchInclude() for matching include patterns against relative paths - Add ShouldDescend() for directory descent via direct match or prefix match - Add IncludePatterns field to GenerateConfig - Validate patterns with doublestar.PathMatch on construction - ** now supports recursive directory matching in exclude patterns Signed-off-by: Zhao Chen Signed-off-by: Zhao Chen --- pkg/config/modelfile/modelfile.go | 2 + pkg/modelfile/modelfile.go | 2 +- pkg/modelfile/path_filter.go | 87 ++++++++-- pkg/modelfile/path_filter_test.go | 270 +++++++++++++++++++++++++++--- 4 files changed, 325 insertions(+), 36 deletions(-) diff --git a/pkg/config/modelfile/modelfile.go b/pkg/config/modelfile/modelfile.go index a51789f..7e06192 100644 --- a/pkg/config/modelfile/modelfile.go +++ b/pkg/config/modelfile/modelfile.go @@ -43,6 +43,7 @@ type GenerateConfig struct { Provider string // Explicit provider for short-form URLs (e.g., "huggingface", "modelscope") DownloadDir string // Custom directory for downloading models (optional) ExcludePatterns []string + IncludePatterns []string } func NewGenerateConfig() *GenerateConfig { @@ -63,6 +64,7 @@ func NewGenerateConfig() *GenerateConfig { Provider: "", DownloadDir: "", ExcludePatterns: []string{}, + IncludePatterns: []string{}, } } diff --git a/pkg/modelfile/modelfile.go b/pkg/modelfile/modelfile.go index dd2604f..971a63d 100644 --- a/pkg/modelfile/modelfile.go +++ b/pkg/modelfile/modelfile.go @@ -258,7 +258,7 @@ func (mf *modelfile) generateByWorkspace(config *configmodelfile.GenerateConfig) var totalSize int64 // Initialize exclude patterns - filter, err := NewPathFilter(config.ExcludePatterns...) + filter, err := NewPathFilter(config.ExcludePatterns, config.IncludePatterns) if err != nil { return err } diff --git a/pkg/modelfile/path_filter.go b/pkg/modelfile/path_filter.go index c742920..5c5f8a9 100644 --- a/pkg/modelfile/path_filter.go +++ b/pkg/modelfile/path_filter.go @@ -4,35 +4,66 @@ import ( "fmt" "path/filepath" "strings" + + "github.com/bmatcuk/doublestar/v4" ) type PathFilter struct { - patterns []string + excludePatterns []string + includePatterns []string } -func NewPathFilter(patterns ...string) (*PathFilter, error) { - var cleaned []string - for _, p := range patterns { - // validate the pattern - if _, err := filepath.Match(p, ""); err != nil { - return nil, fmt.Errorf("invalid exclude pattern: %q", p) +func NewPathFilter(excludePatterns []string, includePatterns []string) (*PathFilter, error) { + var cleanedExclude []string + for _, p := range excludePatterns { + if _, err := doublestar.PathMatch(p, ""); err != nil { + return nil, fmt.Errorf("invalid exclude pattern %q: %w", p, err) + } + cleanedExclude = append(cleanedExclude, strings.TrimRight(p, string(filepath.Separator))) + } + + var cleanedInclude []string + for _, p := range includePatterns { + if _, err := doublestar.PathMatch(p, ""); err != nil { + return nil, fmt.Errorf("invalid include pattern %q: %w", p, err) } - // since filepath.Walk never returns a path with trailing separator, we need to remove separator from patterns - cleaned = append(cleaned, strings.TrimRight(p, string(filepath.Separator))) + cleanedInclude = append(cleanedInclude, strings.TrimRight(p, string(filepath.Separator))) } - return &PathFilter{patterns: cleaned}, nil + + return &PathFilter{ + excludePatterns: cleanedExclude, + includePatterns: cleanedInclude, + }, nil } +// Match checks if a path matches any exclude pattern. func (pf *PathFilter) Match(path string) bool { - if len(pf.patterns) == 0 { + if len(pf.excludePatterns) == 0 { + return false + } + + for _, pattern := range pf.excludePatterns { + matched, err := doublestar.PathMatch(pattern, path) + if err != nil { + return false + } + if matched { + return true + } + } + + return false +} + +// MatchInclude checks if a relative path matches any include pattern. +func (pf *PathFilter) MatchInclude(relPath string) bool { + if len(pf.includePatterns) == 0 { return false } - for _, pattern := range pf.patterns { - matched, err := filepath.Match(pattern, path) + for _, pattern := range pf.includePatterns { + matched, err := doublestar.PathMatch(pattern, relPath) if err != nil { - // The only possible returned error is ErrBadPattern - // which we checked when creating the filter return false } if matched { @@ -42,3 +73,29 @@ func (pf *PathFilter) Match(path string) bool { return false } + +// ShouldDescend checks if a skippable directory should be entered +// because an include pattern might match files inside it. +func (pf *PathFilter) ShouldDescend(dirRelPath string) bool { + if len(pf.includePatterns) == 0 { + return false + } + + for _, pattern := range pf.includePatterns { + // Direct match: pattern matches the directory itself + // e.g., "**/.*" matches ".weights" (** matches zero segments) + matched, err := doublestar.PathMatch(pattern, dirRelPath) + if err == nil && matched { + return true + } + + // Prefix match: pattern starts with dirRelPath/ + // e.g., ".weights/**" starts with ".weights/" + prefix := dirRelPath + string(filepath.Separator) + if strings.HasPrefix(pattern, prefix) { + return true + } + } + + return false +} diff --git a/pkg/modelfile/path_filter_test.go b/pkg/modelfile/path_filter_test.go index 2e18ef1..394f097 100644 --- a/pkg/modelfile/path_filter_test.go +++ b/pkg/modelfile/path_filter_test.go @@ -10,44 +10,62 @@ import ( func TestNewPathFilter(t *testing.T) { testcases := []struct { - name string - input []string - expected []string - expectError bool - errorMsg string + name string + exclude []string + include []string + expectedExclude []string + expectedInclude []string + expectError bool + errorMsg string }{ { - name: "normal patterns", - input: []string{"*.log", "checkpoint*/"}, - expected: []string{"*.log", "checkpoint*"}, + name: "normal exclude patterns", + exclude: []string{"*.log", "checkpoint*/"}, + expectedExclude: []string{"*.log", "checkpoint*"}, }, { - name: "invalid pattern", - input: []string{"*.log", "[invalid"}, + name: "invalid exclude pattern", + exclude: []string{"*.log", "[invalid"}, expectError: true, - errorMsg: `invalid exclude pattern: "[invalid"`, + errorMsg: `invalid exclude pattern "[invalid"`, + }, + { + name: "valid include patterns", + include: []string{"**/.*", ".weights/**"}, + expectedInclude: []string{"**/.*", ".weights/**"}, + }, + { + name: "invalid include pattern", + include: []string{"[invalid"}, + expectError: true, + errorMsg: `invalid include pattern "[invalid"`, }, } for _, tc := range testcases { t.Run(tc.name, func(t *testing.T) { - filter, err := NewPathFilter(tc.input...) + filter, err := NewPathFilter(tc.exclude, tc.include) if tc.expectError { - require.Error(t, err, "Expected an error for input: %q", tc.input) + require.Error(t, err, "Expected an error") assert.Contains(t, err.Error(), tc.errorMsg) assert.Nil(t, filter) return } - require.NoError(t, err, "Did not expect an error for input: %q", tc.input) + require.NoError(t, err) require.NotNil(t, filter) - assert.Equal(t, tc.expected, filter.patterns) + if tc.expectedExclude != nil { + assert.Equal(t, tc.expectedExclude, filter.excludePatterns) + } + if tc.expectedInclude != nil { + assert.Equal(t, tc.expectedInclude, filter.includePatterns) + } }) } } -func TestPathFilter_Matches(t *testing.T) { +func TestPathFilter_Match(t *testing.T) { testcases := []struct { filterName string patterns []string @@ -100,7 +118,7 @@ func TestPathFilter_Matches(t *testing.T) { }, }, { - // Since filepath.Match does not support **, the behavior is the same as Directory_Wildcard_Filter + // With doublestar, ** now matches across path separators filterName: "Directory_Double_Asterisk_Filter", patterns: []string{"build/**"}, tests: []struct { @@ -110,8 +128,22 @@ func TestPathFilter_Matches(t *testing.T) { }{ {"matches file directly inside", "build/app", true}, {"matches hidden file inside", "build/.config", true}, - {"does not match the directory itself", "build", false}, - {"does not match nested files", "build/assets/icon.png", false}, + {"matches the directory itself with doublestar", "build", true}, + {"matches nested files with doublestar", "build/assets/icon.png", true}, + }, + }, + { + filterName: "Recursive_Glob_Filter", + patterns: []string{"**/*.log"}, + tests: []struct { + desc string + path string + expected bool + }{ + {"matches root level log", "debug.log", true}, + {"matches nested log", "subdir/debug.log", true}, + {"matches deeply nested log", "a/b/c/debug.log", true}, + {"does not match non-log", "main.go", false}, }, }, { @@ -147,7 +179,7 @@ func TestPathFilter_Matches(t *testing.T) { for _, tc := range testcases { t.Run(tc.filterName, func(t *testing.T) { - filter, err := NewPathFilter(tc.patterns...) + filter, err := NewPathFilter(tc.patterns, nil) require.NoError(t, err, "Filter creation with patterns %q failed", tc.patterns) require.NotNil(t, filter) @@ -160,3 +192,201 @@ func TestPathFilter_Matches(t *testing.T) { }) } } + +func TestPathFilter_MatchInclude(t *testing.T) { + testcases := []struct { + name string + includes []string + tests []struct { + desc string + path string + expected bool + } + }{ + { + name: "empty include patterns", + includes: nil, + tests: []struct { + desc string + path string + expected bool + }{ + {"any path returns false", "subdir/.hidden", false}, + {"root hidden returns false", ".hidden", false}, + }, + }, + { + name: "root-only dot pattern", + includes: []string{".*"}, + tests: []struct { + desc string + path string + expected bool + }{ + {"matches root hidden file", ".hidden", true}, + {"matches root .env", ".env", true}, + {"does not match nested hidden", "subdir/.hidden", false}, + {"does not match normal file", "normal.txt", false}, + }, + }, + { + name: "recursive dot pattern", + includes: []string{"**/.*"}, + tests: []struct { + desc string + path string + expected bool + }{ + {"matches root hidden", ".hidden", true}, + {"matches nested hidden", "subdir/.hidden", true}, + {"matches deeply nested", "a/b/.config", true}, + {"does not match normal", "subdir/normal", false}, + }, + }, + { + name: "specific dir pattern", + includes: []string{".weights/**"}, + tests: []struct { + desc string + path string + expected bool + }{ + {"matches file inside", ".weights/model.bin", true}, + {"matches nested file inside", ".weights/sub/data.bin", true}, + {"does not match other hidden", ".config/file", false}, + }, + }, + { + name: "brace expansion", + includes: []string{".{weights,config}/**"}, + tests: []struct { + desc string + path string + expected bool + }{ + {"matches .weights/x", ".weights/model.bin", true}, + {"matches .config/y", ".config/settings.json", true}, + {"does not match .other/z", ".other/file", false}, + }, + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + filter, err := NewPathFilter(nil, tc.includes) + require.NoError(t, err) + + for _, test := range tc.tests { + t.Run(test.desc, func(t *testing.T) { + result := filter.MatchInclude(test.path) + assert.Equal(t, test.expected, result, "Path: %q", test.path) + }) + } + }) + } +} + +func TestPathFilter_ShouldDescend(t *testing.T) { + testcases := []struct { + name string + includes []string + tests []struct { + desc string + dirPath string + expected bool + } + }{ + { + name: "no include patterns", + includes: nil, + tests: []struct { + desc string + dirPath string + expected bool + }{ + {"any dir returns false", ".weights", false}, + }, + }, + { + name: "recursive dot pattern enters dot dirs only", + includes: []string{"**/.*"}, + tests: []struct { + desc string + dirPath string + expected bool + }{ + {"enters root hidden dir", ".weights", true}, + {"enters nested hidden dir", "subdir/.hidden", true}, + {"does not enter __pycache__", "__pycache__", false}, + {"does not enter normal skippable", "modelfile", false}, + }, + }, + { + name: "pycache pattern enters pycache only", + includes: []string{"**/__pycache__/**"}, + tests: []struct { + desc string + dirPath string + expected bool + }{ + {"enters __pycache__", "__pycache__", true}, + {"enters nested __pycache__", "src/__pycache__", true}, + {"does not enter hidden dir", ".hidden", false}, + }, + }, + { + name: "prefix match for specific dir", + includes: []string{".weights/**"}, + tests: []struct { + desc string + dirPath string + expected bool + }{ + {"enters .weights", ".weights", true}, + {"does not enter .weird", ".weird", false}, + {"does not enter .config", ".config", false}, + }, + }, + { + name: "prefix boundary", + includes: []string{".weights/*"}, + tests: []struct { + desc string + dirPath string + expected bool + }{ + {"enters .weights", ".weights", true}, + {"does not enter .weird", ".weird", false}, + }, + }, + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + filter, err := NewPathFilter(nil, tc.includes) + require.NoError(t, err) + + for _, test := range tc.tests { + t.Run(test.desc, func(t *testing.T) { + result := filter.ShouldDescend(test.dirPath) + assert.Equal(t, test.expected, result, "Dir: %q", test.dirPath) + }) + } + }) + } +} + +func TestPathFilter_IncludeExcludeInteraction(t *testing.T) { + // File matches both include and exclude → excluded (exclude wins) + filter, err := NewPathFilter([]string{"**/.env"}, []string{"**/.*"}) + require.NoError(t, err) + + // .env matches include pattern **/.* + assert.True(t, filter.MatchInclude(".env")) + // .env also matches exclude pattern **/.env + assert.True(t, filter.Match(".env")) + + // .hidden matches include but not exclude + assert.True(t, filter.MatchInclude(".hidden")) + assert.False(t, filter.Match(".hidden")) +} From ee6d019f8b00519f1abaedb02543e4e8dc252d47 Mon Sep 17 00:00:00 2001 From: Zhao Chen Date: Thu, 26 Mar 2026 18:33:21 +0800 Subject: [PATCH 2/6] feat: add --include flag to modelfile generate command Bind --include flag with doublestar glob syntax. Help text documents pattern syntax, matching base (relative to workspace root), and risk warning for broad patterns. Signed-off-by: Zhao Chen Signed-off-by: Zhao Chen --- cmd/modelfile/generate.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cmd/modelfile/generate.go b/cmd/modelfile/generate.go index 7f2e257..74a52c3 100644 --- a/cmd/modelfile/generate.go +++ b/cmd/modelfile/generate.go @@ -112,6 +112,10 @@ func init() { flags.StringVarP(&generateConfig.Provider, "provider", "p", "", "explicitly specify the provider for short-form URLs (huggingface, modelscope)") flags.StringVar(&generateConfig.DownloadDir, "download-dir", "", "custom directory for downloading models (default: system temp directory)") flags.StringArrayVar(&generateConfig.ExcludePatterns, "exclude", []string{}, "specify glob patterns to exclude files/directories (e.g. *.log, checkpoints/*)") + flags.StringArrayVar(&generateConfig.IncludePatterns, "include", []string{}, + "glob patterns to include files/directories that are normally skipped (e.g. hidden files).\n"+ + "Uses doublestar syntax (*, **, ?, [...], {a,b}), matching against relative paths from workspace root.\n"+ + "Note: broad patterns like **/.* may include large directories (.git) or sensitive files (.env)") // Mark the ignore-unrecognized-file-types flag as deprecated and hidden flags.MarkDeprecated("ignore-unrecognized-file-types", "this flag will be removed in the next release") From 4f52ef5a4f7b0a4f131ff8d3fac1599412eb3a6a Mon Sep 17 00:00:00 2001 From: Zhao Chen Date: Thu, 26 Mar 2026 18:34:11 +0800 Subject: [PATCH 3/6] feat: integrate include pattern logic into generateByWorkspace Update filter logic: directory exclude is absolute, isSkippable entries can be rescued by include patterns via ShouldDescend and MatchInclude, rescued files still go through exclude check. Signed-off-by: Zhao Chen Signed-off-by: Zhao Chen --- pkg/modelfile/modelfile.go | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/pkg/modelfile/modelfile.go b/pkg/modelfile/modelfile.go index 971a63d..264ba96 100644 --- a/pkg/modelfile/modelfile.go +++ b/pkg/modelfile/modelfile.go @@ -277,12 +277,32 @@ func (mf *modelfile) generateByWorkspace(config *configmodelfile.GenerateConfig) return err } - // Skip hidden, skippable, and excluded files/directories. - if isSkippable(filename) || filter.Match(relPath) { + // Directory exclude is absolute — cannot be reversed by --include. + if info.IsDir() && filter.Match(relPath) { + return filepath.SkipDir + } + + // Check skipPatterns — include can rescue skippable entries. + if isSkippable(filename) { if info.IsDir() { - return filepath.SkipDir + if filter.ShouldDescend(relPath) { + // Rescued by --include, enter directory + } else { + return filepath.SkipDir + } + } else { + if !filter.MatchInclude(relPath) { + return nil + } + // Rescued file still goes through exclude check below } + } + // Exclude check for non-skippable files (and include-rescued files). + if filter.Match(relPath) { + if info.IsDir() { + return filepath.SkipDir + } return nil } From ee894af89f5cf6abc98ae8b88de31adc736d3f2f Mon Sep 17 00:00:00 2001 From: Zhao Chen Date: Thu, 26 Mar 2026 18:37:52 +0800 Subject: [PATCH 4/6] test: add integration tests for --include flag Cover recursive hidden files, specific directory include, include with exclude override, regression (no include), multiple patterns, and selective directory entry. Signed-off-by: Zhao Chen Signed-off-by: Zhao Chen --- pkg/modelfile/modelfile_test.go | 135 ++++++++++++++++++++++++++++++++ 1 file changed, 135 insertions(+) diff --git a/pkg/modelfile/modelfile_test.go b/pkg/modelfile/modelfile_test.go index 1c4fefb..038219e 100644 --- a/pkg/modelfile/modelfile_test.go +++ b/pkg/modelfile/modelfile_test.go @@ -676,6 +676,141 @@ func TestNewModelfileByWorkspace(t *testing.T) { expectCodes: []string{"valid_dir/model.py"}, expectName: "skip-test", }, + { + name: "include all hidden files with recursive pattern", + setupFiles: map[string]string{ + "config.json": "", + "model.bin": "", + ".hidden_config.json": "", + ".hidden_dir/model.bin": "", + ".hidden_dir/.nested.py": "", + "normal_dir/.hidden_code.py": "", + "normal_dir/visible.py": "", + }, + setupDirs: []string{ + ".hidden_dir", + "normal_dir", + }, + config: &configmodelfile.GenerateConfig{ + Name: "include-all-hidden", + IncludePatterns: []string{"**/.*"}, + }, + expectError: false, + expectConfigs: []string{"config.json", ".hidden_config.json"}, + expectModels: []string{"model.bin", ".hidden_dir/model.bin"}, + expectCodes: []string{".hidden_dir/.nested.py", "normal_dir/.hidden_code.py", "normal_dir/visible.py"}, + expectName: "include-all-hidden", + }, + { + name: "include specific hidden directory", + setupFiles: map[string]string{ + "config.json": "", + "model.bin": "", + ".weights/extra.bin": "", + ".weights/data.bin": "", + ".other/secret.py": "", + }, + setupDirs: []string{ + ".weights", + ".other", + }, + config: &configmodelfile.GenerateConfig{ + Name: "include-weights-dir", + IncludePatterns: []string{".weights/**"}, + }, + expectError: false, + expectConfigs: []string{"config.json"}, + expectModels: []string{"model.bin", ".weights/extra.bin", ".weights/data.bin"}, + expectCodes: []string{}, + expectName: "include-weights-dir", + }, + { + name: "include with exclude override", + setupFiles: map[string]string{ + "config.json": "", + "model.bin": "", + ".hidden.py": "", + ".env": "", + "sub/.secret.yml": "", + }, + setupDirs: []string{ + "sub", + }, + config: &configmodelfile.GenerateConfig{ + Name: "include-exclude", + IncludePatterns: []string{"**/.*"}, + ExcludePatterns: []string{"**/.env"}, + }, + expectError: false, + expectConfigs: []string{"config.json", "sub/.secret.yml"}, + expectModels: []string{"model.bin"}, + expectCodes: []string{".hidden.py"}, + expectName: "include-exclude", + }, + { + name: "no include patterns regression", + setupFiles: map[string]string{ + "config.json": "", + "model.bin": "", + ".hidden_file": "", + ".hidden_dir/x.py": "", + }, + setupDirs: []string{ + ".hidden_dir", + }, + config: &configmodelfile.GenerateConfig{ + Name: "no-include-regression", + }, + expectError: false, + expectConfigs: []string{"config.json"}, + expectModels: []string{"model.bin"}, + expectCodes: []string{}, + expectName: "no-include-regression", + }, + { + name: "multiple include patterns", + setupFiles: map[string]string{ + "config.json": "", + "model.bin": "", + ".hidden.py": "", + "__pycache__/cache.pyc": "", + }, + setupDirs: []string{ + "__pycache__", + }, + config: &configmodelfile.GenerateConfig{ + Name: "multi-include", + IncludePatterns: []string{".*", "**/__pycache__/**"}, + }, + expectError: false, + expectConfigs: []string{"config.json"}, + expectModels: []string{"model.bin"}, + expectCodes: []string{".hidden.py", "__pycache__/cache.pyc"}, + expectName: "multi-include", + }, + { + name: "skippable dirs not matching include are still skipped", + setupFiles: map[string]string{ + "config.json": "", + "model.bin": "", + ".git/objects/pack": "", + ".weights/model.bin": "", + }, + setupDirs: []string{ + ".git", + ".git/objects", + ".weights", + }, + config: &configmodelfile.GenerateConfig{ + Name: "selective-include", + IncludePatterns: []string{".weights/**"}, + }, + expectError: false, + expectConfigs: []string{"config.json"}, + expectModels: []string{"model.bin", ".weights/model.bin"}, + expectCodes: []string{}, + expectName: "selective-include", + }, } assert := assert.New(t) From c3dea59eeab6e7dd3c05238fa11d29c6c4230fd3 Mon Sep 17 00:00:00 2001 From: Zhao Chen Date: Thu, 26 Mar 2026 18:48:49 +0800 Subject: [PATCH 5/6] style: apply linter formatting fixes Signed-off-by: Zhao Chen Signed-off-by: Zhao Chen --- pkg/modelfile/constants.go | 88 +++++++++++++++---------------- pkg/modelfile/modelfile_test.go | 22 ++++---- pkg/modelfile/path_filter_test.go | 14 ++--- 3 files changed, 62 insertions(+), 62 deletions(-) diff --git a/pkg/modelfile/constants.go b/pkg/modelfile/constants.go index bdf16b9..8ec3df2 100644 --- a/pkg/modelfile/constants.go +++ b/pkg/modelfile/constants.go @@ -65,12 +65,12 @@ var ( // PyTorch formats. "*.bin", // General binary format "*.bin.*", // Sharded binary files (e.g., model.bin.1) - "*.pt", // PyTorch model - "*.pth", // PyTorch model (alternative extension) - "*.mar", // PyTorch Model Archive - "*.pte", // PyTorch ExecuTorch format - "*.pt2", // PyTorch 2.0 export format - "*.ptl", // PyTorch Mobile format + "*.pt", // PyTorch model + "*.pth", // PyTorch model (alternative extension) + "*.mar", // PyTorch Model Archive + "*.pte", // PyTorch ExecuTorch format + "*.pt2", // PyTorch 2.0 export format + "*.ptl", // PyTorch Mobile format // TensorFlow formats. "*.tflite", // TensorFlow Lite @@ -85,16 +85,16 @@ var ( // GGML formats. "*.gguf", // GGML Universal Format "*.gguf.*", // Partitioned GGUF files - "*.ggml", // GGML format (legacy) - "*.ggmf", // GGMF format (deprecated) - "*.ggjt", // GGJT format (deprecated) - "*.q4_0", // GGML Q4_0 quantization - "*.q4_1", // GGML Q4_1 quantization - "*.q5_0", // GGML Q5_0 quantization - "*.q5_1", // GGML Q5_1 quantization - "*.q8_0", // GGML Q8_0 quantization - "*.f16", // GGML F16 format - "*.f32", // GGML F32 format + "*.ggml", // GGML format (legacy) + "*.ggmf", // GGMF format (deprecated) + "*.ggjt", // GGJT format (deprecated) + "*.q4_0", // GGML Q4_0 quantization + "*.q4_1", // GGML Q4_1 quantization + "*.q5_0", // GGML Q5_0 quantization + "*.q5_1", // GGML Q5_1 quantization + "*.q8_0", // GGML Q8_0 quantization + "*.f16", // GGML F16 format + "*.f32", // GGML F32 format // checkpoint formats. "*.ckpt", // Checkpoint format @@ -109,37 +109,37 @@ var ( "*.vocab", // Vocabulary files (when binary) // Other ML frameworks. - "*.ot", // OpenVINO format - "*.engine", // TensorRT format - "*.trt", // TensorRT format (alternative extension) - "*.onnx", // Open Neural Network Exchange format - "*.msgpack", // MessagePack serialization - "*.model", // Some NLP frameworks - "*.pkl", // Pickle format - "*.pickle", // Pickle format (alternative extension) - "*.keras", // Keras native format - "*.joblib", // Joblib serialization (scikit-learn) - "*.npy", // NumPy array format - "*.npz", // NumPy compressed archive - "*.nc", // NetCDF format - "*.mlmodel", // Apple Core ML format - "*.coreml", // Apple Core ML format (alternative) - "*.mleap", // MLeap format (Spark ML) - "*.surml", // SurrealML format - "*.llamafile", // Llamafile format + "*.ot", // OpenVINO format + "*.engine", // TensorRT format + "*.trt", // TensorRT format (alternative extension) + "*.onnx", // Open Neural Network Exchange format + "*.msgpack", // MessagePack serialization + "*.model", // Some NLP frameworks + "*.pkl", // Pickle format + "*.pickle", // Pickle format (alternative extension) + "*.keras", // Keras native format + "*.joblib", // Joblib serialization (scikit-learn) + "*.npy", // NumPy array format + "*.npz", // NumPy compressed archive + "*.nc", // NetCDF format + "*.mlmodel", // Apple Core ML format + "*.coreml", // Apple Core ML format (alternative) + "*.mleap", // MLeap format (Spark ML) + "*.surml", // SurrealML format + "*.llamafile", // Llamafile format "*.llamafile.*", // Llamafile variants - "*.caffemodel", // Caffe model format - "*.prototxt", // Caffe model definition - "*.dlc", // Qualcomm Deep Learning Container - "*.circle", // Samsung Circle format - "*.nb", // Neural Network Binary format + "*.caffemodel", // Caffe model format + "*.prototxt", // Caffe model definition + "*.dlc", // Qualcomm Deep Learning Container + "*.circle", // Samsung Circle format + "*.nb", // Neural Network Binary format // Data and dataset formats. - "*.arrow", // Apache Arrow columnar format - "*.parquet", // Apache Parquet columnar format - "*.ftz", // FastText compressed model - "*.ark", // Kaldi ark format (speech/audio models) - "*.db", // Database files (LMDB, etc.) + "*.arrow", // Apache Arrow columnar format + "*.parquet", // Apache Parquet columnar format + "*.ftz", // FastText compressed model + "*.ark", // Kaldi ark format (speech/audio models) + "*.db", // Database files (LMDB, etc.) } // Code file patterns - supported script and notebook files. diff --git a/pkg/modelfile/modelfile_test.go b/pkg/modelfile/modelfile_test.go index 038219e..b9c3b08 100644 --- a/pkg/modelfile/modelfile_test.go +++ b/pkg/modelfile/modelfile_test.go @@ -679,13 +679,13 @@ func TestNewModelfileByWorkspace(t *testing.T) { { name: "include all hidden files with recursive pattern", setupFiles: map[string]string{ - "config.json": "", - "model.bin": "", - ".hidden_config.json": "", - ".hidden_dir/model.bin": "", - ".hidden_dir/.nested.py": "", - "normal_dir/.hidden_code.py": "", - "normal_dir/visible.py": "", + "config.json": "", + "model.bin": "", + ".hidden_config.json": "", + ".hidden_dir/model.bin": "", + ".hidden_dir/.nested.py": "", + "normal_dir/.hidden_code.py": "", + "normal_dir/visible.py": "", }, setupDirs: []string{ ".hidden_dir", @@ -750,10 +750,10 @@ func TestNewModelfileByWorkspace(t *testing.T) { { name: "no include patterns regression", setupFiles: map[string]string{ - "config.json": "", - "model.bin": "", - ".hidden_file": "", - ".hidden_dir/x.py": "", + "config.json": "", + "model.bin": "", + ".hidden_file": "", + ".hidden_dir/x.py": "", }, setupDirs: []string{ ".hidden_dir", diff --git a/pkg/modelfile/path_filter_test.go b/pkg/modelfile/path_filter_test.go index 394f097..71107aa 100644 --- a/pkg/modelfile/path_filter_test.go +++ b/pkg/modelfile/path_filter_test.go @@ -10,13 +10,13 @@ import ( func TestNewPathFilter(t *testing.T) { testcases := []struct { - name string - exclude []string - include []string - expectedExclude []string - expectedInclude []string - expectError bool - errorMsg string + name string + exclude []string + include []string + expectedExclude []string + expectedInclude []string + expectError bool + errorMsg string }{ { name: "normal exclude patterns", From 483b89d3276857c9b82cb12de8b60dc43032f28f Mon Sep 17 00:00:00 2001 From: Zhao Chen Date: Thu, 26 Mar 2026 18:47:39 +0800 Subject: [PATCH 6/6] docs: add --include examples to modelfile generate help Signed-off-by: Zhao Chen Signed-off-by: Zhao Chen --- cmd/modelfile/generate.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/cmd/modelfile/generate.go b/cmd/modelfile/generate.go index 74a52c3..4317b58 100644 --- a/cmd/modelfile/generate.go +++ b/cmd/modelfile/generate.go @@ -64,7 +64,16 @@ Full URLs with domain names will auto-detect the provider.`, modctl modelfile generate ./my-model-dir --output ./output/modelfile.yaml # Generate with metadata overrides - modctl modelfile generate ./my-model-dir --name my-custom-model --family llama3`, + modctl modelfile generate ./my-model-dir --name my-custom-model --family llama3 + + # Include hidden files at any depth + modctl modelfile generate ./my-model-dir --include "**/.*" + + # Include a specific hidden directory + modctl modelfile generate ./my-model-dir --include ".weights/**" + + # Include hidden files but exclude sensitive ones + modctl modelfile generate ./my-model-dir --include "**/.*" --exclude "**/.env"`, Args: cobra.MaximumNArgs(1), DisableAutoGenTag: true, SilenceUsage: true,